Merge pull request #9 from dscripka/voice_activity_detection

Voice activity detection

openwakeword/__init__.py

@@ -1,7 +1,8 @@
 import os
 from openwakeword.model import Model
+from openwakeword.vad import VAD
 
-__all__ = ['Model']
+__all__ = ['Model', 'VAD']
 
 models = {
     "alexa": {

openwakeword/model.py

@@ -37,6 +37,7 @@ class Model():
             wakeword_model_paths: List[str] = [],
             class_mapping_dicts: List[dict] = [],
             enable_speex_noise_suppression: bool = False,
+            vad_threshold: float = 0,
             **kwargs
             ):
         """Initialize the openWakeWord model object.
@@ -53,6 +54,11 @@ class Model():
                                                    is present in the environment where openWakeWord will be used.
                                                    It is very lightweight, so enabling it doesn't significantly
                                                    impact efficiency.
+            vad_threshold (float): Whether to use a voice activity detection model (VAD) from Silero
+                                   (https://github.com/snakers4/silero-vad) to filter predictions.
+                                   For every input audio frame, a VAD score is obtained and only those model predictions
+                                   with VAD scores above the threshold will be returned. The default value (0),
+                                   disables voice activity detection entirely.
         """
 
         # Initialize the ONNX models and store them
@@ -96,6 +102,11 @@ class Model():
         else:
             self.speex_ns = None
 
+        # Initialize Silero VAD
+        self.vad_threshold = vad_threshold
+        if vad_threshold > 0:
+            self.vad = openwakeword.VAD()
+
         # Create AudioFeatures object
         self.preprocessor = AudioFeatures(**kwargs)
 
@@ -114,7 +125,7 @@ class Model():
         """Reset the prediction buffer"""
         self.prediction_buffer = defaultdict(partial(deque, maxlen=30))
 
-    def predict(self, x: Union[np.ndarray], patience: dict = {}, threshold: dict = {}, timing: bool = False):
+    def predict(self, x: np.ndarray, patience: dict = {}, threshold: dict = {}, timing: bool = False):
         """Predict with all of the wakeword models on the input audio frames
 
         Args:
@@ -137,20 +148,21 @@ class Model():
                   wake-word/wake-phrase detected. If the `timing` argument is true, returns a
                   tuple of dicts containing model predictions and timing information, respectively.
         """
-        # Get audio features (optionally with Speex noise suppression)
+
+        # Setup timing dict
         if timing:
             timing_dict: Dict[str, Dict] = {}
             timing_dict["models"] = {}
             feature_start = time.time()
 
+        # Get audio features (optionally with Speex noise suppression)
         if self.speex_ns:
             self.preprocessor(self._suppress_noise_with_speex(x))
         else:
             self.preprocessor(x)
 
         if timing:
-            feature_end = time.time()
-            timing_dict["models"]["preprocessor"] = feature_end - feature_start
+            timing_dict["models"]["preprocessor"] = time.time() - feature_start
 
         # Get predictions from model(s)
         predictions = {}
@@ -179,8 +191,7 @@ class Model():
 
             # Get timing information
             if timing:
-                model_end = time.time()
-                timing_dict["models"][mdl] = model_end - model_start
+                timing_dict["models"][mdl] = time.time() - model_start
 
         # Update scores based on thresholds or patience arguments
         if patience != {}:
@@ -194,6 +205,23 @@ class Model():
                     if (scores >= threshold[parent_model]).sum() < patience[parent_model]:
                         predictions[mdl] = 0.0
 
+        # (optionally) get voice activity detection scores and update model scores
+        if self.vad_threshold > 0:
+            if timing:
+                vad_start = time.time()
+
+            self.vad(x)
+
+            if timing:
+                timing_dict["models"]["vad"] = time.time() - vad_start
+
+            # Get frames from last 0.4 to 0.56 seconds (3 frames) and get max VAD score
+            vad_frames = list(self.vad.prediction_buffer)[-7:-4]
+            vad_avg_score = np.max(vad_frames) if len(vad_frames) > 0 else 0
+            for mdl in predictions.keys():
+                if vad_avg_score < self.vad_threshold:
+                    predictions[mdl] = 0.0
+
         if timing:
             return predictions, timing_dict
         else:

openwakeword/resources/models/silero_vad.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a35ebf52fd3ce5f1469b2a36158dba761bc47b973ea3382b3186ca15b1f5af28
+size 1807522

openwakeword/vad.py

@@ -0,0 +1,128 @@
+# Copyright 2022 David Scripka. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#######################
+# Silero VAD License
+#######################
+
+# MIT License
+
+# Copyright (c) 2020-present Silero Team
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+########################################
+
+# This file contains the implementation of a class for voice activity detection (VAD),
+# based on the pre-trained model from Silero (https://github.com/snakers4/silero-vad).
+# It can be used as with the openWakeWord library, or independently.
+
+# Imports
+import onnxruntime as ort
+import numpy as np
+import os
+from collections import deque
+
+
+class VAD():
+    """
+    A model class for a voice activity detection (VAD) based on Silero's model:
+
+    https://github.com/snakers4/silero-vad
+    """
+    def __init__(self,
+                 model_path: str = os.path.join(
+                    os.path.dirname(os.path.abspath(__file__)),
+                    "resources",
+                    "models",
+                    "silero_vad.onnx"
+                 )
+                 ):
+        """Initialize the VAD model object.
+
+            Args:
+                model_path (str): The path to the Silero VAD ONNX model.
+        """
+
+        # Initialize the ONNX model
+        sessionOptions = ort.SessionOptions()
+        sessionOptions.inter_op_num_threads = 1
+        sessionOptions.intra_op_num_threads = 1
+        self.model = ort.InferenceSession(model_path, sess_options=sessionOptions,
+                                          providers=["CPUExecutionProvider"])
+
+        # Create buffer
+        self.prediction_buffer: deque = deque(maxlen=125)  # buffer lenght of 10 seconds
+
+        # Set model parameters
+        self.sample_rate = np.array(16000).astype(np.int64)
+
+        # Reset model to start
+        self.reset_states()
+
+    def reset_states(self, batch_size=1):
+        self._h = np.zeros((2, batch_size, 64)).astype('float32')
+        self._c = np.zeros((2, batch_size, 64)).astype('float32')
+        self._last_sr = 0
+        self._last_batch_size = 0
+
+    def predict(self, x, frame_size=480):
+        """
+        Get the VAD predictions for the input audio frame.
+
+        Args:
+            x (np.ndarray): The input audio, must be 16 khz and 16-bit PCM format.
+                            If longer than the input frame, will be split into
+                            chunks of length `frame_size` and the predictions for
+                            each chunk returned. Must be a length that is integer
+                            multiples of the `frame_size` argument.
+            frame_size (int): The frame size in samples. The reccomended
+                              default is 480 samples (30 ms @ 16khz),
+                              but smaller and larger values
+                              can be used (though performance may decrease).
+
+        Returns
+            float: The average predicted score for the audio frame
+        """
+        chunks = [(x[i:i+frame_size]/32767).astype(np.float32)
+                  for i in range(0, x.shape[0], frame_size)]
+
+        frame_predictions = []
+        for chunk in chunks:
+            ort_inputs = {'input': chunk[None, ],
+                          'h': self._h, 'c': self._c, 'sr': self.sample_rate}
+            ort_outs = self.model.run(None, ort_inputs)
+            out, self._h, self._c = ort_outs
+            frame_predictions.append(out[0][0])
+
+        return np.mean(frame_predictions)
+
+    def __call__(self, x, frame_size=160*4):
+        self.prediction_buffer.append(self.predict(x, frame_size))

tests/test_models.py

@@ -118,6 +118,34 @@ class TestModels:
                     else:
                         assert max(predictions_flat[key]) < 0.5
 
+    def test_models_with_vad(self):
+        # Load model with defaults
+        owwModel = openwakeword.Model(vad_threshold=0.5)
+
+        # Get clips for each model (assumes that test clips will have the model name in the filename)
+        test_dict = {}
+        for mdl_name in owwModel.models.keys():
+            all_clips = [str(i) for i in Path(os.path.join("tests", "data")).glob("*.wav")]
+            test_dict[mdl_name] = [i for i in all_clips if mdl_name in i]
+
+        # Predict
+        for model, clips in test_dict.items():
+            for clip in clips:
+                # Get predictions for reach frame in the clip
+                predictions = owwModel.predict_clip(clip)
+                owwModel.reset()  # reset after each clip to ensure independent results
+
+                # Make predictions dictionary flatter
+                predictions_flat = collections.defaultdict(list)
+                [predictions_flat[key].append(i[key]) for i in predictions for key in i.keys()]
+
+            # Check scores against default threshold (0.5)
+            for key in predictions_flat.keys():
+                if key in clip:
+                    assert max(predictions_flat[key]) >= 0.5
+                else:
+                    assert max(predictions_flat[key]) < 0.5
+
     def test_predict_clip_with_array(self):
         # Load model with defaults
         owwModel = openwakeword.Model()
@@ -129,9 +157,9 @@ class TestModels:
 
     def test_models_with_timing(self):
         # Load model with defaults
-        owwModel = openwakeword.Model()
+        owwModel = openwakeword.Model(vad_threshold=0.5)
 
-        owwModel.predict(np.zeros(1280), timing=True)
+        owwModel.predict(np.zeros(1280).astype(np.int16), timing=True)
 
     def test_prediction_with_patience(self):
         owwModel = openwakeword.Model()