diff --git a/openwakeword/__init__.py b/openwakeword/__init__.py index 718103f..071d58a 100755 --- a/openwakeword/__init__.py +++ b/openwakeword/__init__.py @@ -1,7 +1,8 @@ import os from openwakeword.model import Model +from openwakeword.vad import VAD -__all__ = ['Model'] +__all__ = ['Model', 'VAD'] models = { "alexa": { diff --git a/openwakeword/model.py b/openwakeword/model.py index 9201152..91f1d41 100755 --- a/openwakeword/model.py +++ b/openwakeword/model.py @@ -37,6 +37,7 @@ class Model(): wakeword_model_paths: List[str] = [], class_mapping_dicts: List[dict] = [], enable_speex_noise_suppression: bool = False, + vad_threshold: float = 0, **kwargs ): """Initialize the openWakeWord model object. @@ -53,6 +54,11 @@ class Model(): is present in the environment where openWakeWord will be used. It is very lightweight, so enabling it doesn't significantly impact efficiency. + vad_threshold (float): Whether to use a voice activity detection model (VAD) from Silero + (https://github.com/snakers4/silero-vad) to filter predictions. + For every input audio frame, a VAD score is obtained and only those model predictions + with VAD scores above the threshold will be returned. The default value (0), + disables voice activity detection entirely. """ # Initialize the ONNX models and store them @@ -96,6 +102,11 @@ class Model(): else: self.speex_ns = None + # Initialize Silero VAD + self.vad_threshold = vad_threshold + if vad_threshold > 0: + self.vad = openwakeword.VAD() + # Create AudioFeatures object self.preprocessor = AudioFeatures(**kwargs) @@ -114,7 +125,7 @@ class Model(): """Reset the prediction buffer""" self.prediction_buffer = defaultdict(partial(deque, maxlen=30)) - def predict(self, x: Union[np.ndarray], patience: dict = {}, threshold: dict = {}, timing: bool = False): + def predict(self, x: np.ndarray, patience: dict = {}, threshold: dict = {}, timing: bool = False): """Predict with all of the wakeword models on the input audio frames Args: @@ -137,20 +148,21 @@ class Model(): wake-word/wake-phrase detected. If the `timing` argument is true, returns a tuple of dicts containing model predictions and timing information, respectively. """ - # Get audio features (optionally with Speex noise suppression) + + # Setup timing dict if timing: timing_dict: Dict[str, Dict] = {} timing_dict["models"] = {} feature_start = time.time() + # Get audio features (optionally with Speex noise suppression) if self.speex_ns: self.preprocessor(self._suppress_noise_with_speex(x)) else: self.preprocessor(x) if timing: - feature_end = time.time() - timing_dict["models"]["preprocessor"] = feature_end - feature_start + timing_dict["models"]["preprocessor"] = time.time() - feature_start # Get predictions from model(s) predictions = {} @@ -179,8 +191,7 @@ class Model(): # Get timing information if timing: - model_end = time.time() - timing_dict["models"][mdl] = model_end - model_start + timing_dict["models"][mdl] = time.time() - model_start # Update scores based on thresholds or patience arguments if patience != {}: @@ -194,6 +205,23 @@ class Model(): if (scores >= threshold[parent_model]).sum() < patience[parent_model]: predictions[mdl] = 0.0 + # (optionally) get voice activity detection scores and update model scores + if self.vad_threshold > 0: + if timing: + vad_start = time.time() + + self.vad(x) + + if timing: + timing_dict["models"]["vad"] = time.time() - vad_start + + # Get frames from last 0.4 to 0.56 seconds (3 frames) and get max VAD score + vad_frames = list(self.vad.prediction_buffer)[-7:-4] + vad_avg_score = np.max(vad_frames) if len(vad_frames) > 0 else 0 + for mdl in predictions.keys(): + if vad_avg_score < self.vad_threshold: + predictions[mdl] = 0.0 + if timing: return predictions, timing_dict else: diff --git a/openwakeword/resources/models/silero_vad.onnx b/openwakeword/resources/models/silero_vad.onnx new file mode 100755 index 0000000..664012e --- /dev/null +++ b/openwakeword/resources/models/silero_vad.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a35ebf52fd3ce5f1469b2a36158dba761bc47b973ea3382b3186ca15b1f5af28 +size 1807522 diff --git a/openwakeword/vad.py b/openwakeword/vad.py new file mode 100755 index 0000000..18bf5e3 --- /dev/null +++ b/openwakeword/vad.py @@ -0,0 +1,128 @@ +# Copyright 2022 David Scripka. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +####################### +# Silero VAD License +####################### + +# MIT License + +# Copyright (c) 2020-present Silero Team + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +######################################## + +# This file contains the implementation of a class for voice activity detection (VAD), +# based on the pre-trained model from Silero (https://github.com/snakers4/silero-vad). +# It can be used as with the openWakeWord library, or independently. + +# Imports +import onnxruntime as ort +import numpy as np +import os +from collections import deque + + +class VAD(): + """ + A model class for a voice activity detection (VAD) based on Silero's model: + + https://github.com/snakers4/silero-vad + """ + def __init__(self, + model_path: str = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "resources", + "models", + "silero_vad.onnx" + ) + ): + """Initialize the VAD model object. + + Args: + model_path (str): The path to the Silero VAD ONNX model. + """ + + # Initialize the ONNX model + sessionOptions = ort.SessionOptions() + sessionOptions.inter_op_num_threads = 1 + sessionOptions.intra_op_num_threads = 1 + self.model = ort.InferenceSession(model_path, sess_options=sessionOptions, + providers=["CPUExecutionProvider"]) + + # Create buffer + self.prediction_buffer: deque = deque(maxlen=125) # buffer lenght of 10 seconds + + # Set model parameters + self.sample_rate = np.array(16000).astype(np.int64) + + # Reset model to start + self.reset_states() + + def reset_states(self, batch_size=1): + self._h = np.zeros((2, batch_size, 64)).astype('float32') + self._c = np.zeros((2, batch_size, 64)).astype('float32') + self._last_sr = 0 + self._last_batch_size = 0 + + def predict(self, x, frame_size=480): + """ + Get the VAD predictions for the input audio frame. + + Args: + x (np.ndarray): The input audio, must be 16 khz and 16-bit PCM format. + If longer than the input frame, will be split into + chunks of length `frame_size` and the predictions for + each chunk returned. Must be a length that is integer + multiples of the `frame_size` argument. + frame_size (int): The frame size in samples. The reccomended + default is 480 samples (30 ms @ 16khz), + but smaller and larger values + can be used (though performance may decrease). + + Returns + float: The average predicted score for the audio frame + """ + chunks = [(x[i:i+frame_size]/32767).astype(np.float32) + for i in range(0, x.shape[0], frame_size)] + + frame_predictions = [] + for chunk in chunks: + ort_inputs = {'input': chunk[None, ], + 'h': self._h, 'c': self._c, 'sr': self.sample_rate} + ort_outs = self.model.run(None, ort_inputs) + out, self._h, self._c = ort_outs + frame_predictions.append(out[0][0]) + + return np.mean(frame_predictions) + + def __call__(self, x, frame_size=160*4): + self.prediction_buffer.append(self.predict(x, frame_size)) diff --git a/tests/test_models.py b/tests/test_models.py index 93dcfb4..109ae11 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -118,6 +118,34 @@ class TestModels: else: assert max(predictions_flat[key]) < 0.5 + def test_models_with_vad(self): + # Load model with defaults + owwModel = openwakeword.Model(vad_threshold=0.5) + + # Get clips for each model (assumes that test clips will have the model name in the filename) + test_dict = {} + for mdl_name in owwModel.models.keys(): + all_clips = [str(i) for i in Path(os.path.join("tests", "data")).glob("*.wav")] + test_dict[mdl_name] = [i for i in all_clips if mdl_name in i] + + # Predict + for model, clips in test_dict.items(): + for clip in clips: + # Get predictions for reach frame in the clip + predictions = owwModel.predict_clip(clip) + owwModel.reset() # reset after each clip to ensure independent results + + # Make predictions dictionary flatter + predictions_flat = collections.defaultdict(list) + [predictions_flat[key].append(i[key]) for i in predictions for key in i.keys()] + + # Check scores against default threshold (0.5) + for key in predictions_flat.keys(): + if key in clip: + assert max(predictions_flat[key]) >= 0.5 + else: + assert max(predictions_flat[key]) < 0.5 + def test_predict_clip_with_array(self): # Load model with defaults owwModel = openwakeword.Model() @@ -129,9 +157,9 @@ class TestModels: def test_models_with_timing(self): # Load model with defaults - owwModel = openwakeword.Model() + owwModel = openwakeword.Model(vad_threshold=0.5) - owwModel.predict(np.zeros(1280), timing=True) + owwModel.predict(np.zeros(1280).astype(np.int16), timing=True) def test_prediction_with_patience(self): owwModel = openwakeword.Model()