From 50c1d08ce429f616e6d6a03bf25cc6ffe07c4d41 Mon Sep 17 00:00:00 2001 From: dscripka Date: Sat, 28 Jan 2023 09:40:28 -0500 Subject: [PATCH] Adjusted docs, added first release of new 'hey jarvis' model, small bug fixes in preparation for next release --- README.md | 7 +++- docs/models/alexa.md | 31 ++++++++++++++++++ docs/models/hey_mycroft.md | 31 ++++++++++++++++++ docs/models/timers.md | 29 +++++++++++++++++ docs/models/weather.md | 32 +++++++++++++++++++ openwakeword/metrics.py | 2 +- openwakeword/model.py | 7 ++-- .../resources/models/hey_jarvis_v0.1.onnx | 4 +-- openwakeword/utils.py | 5 +-- 9 files changed, 139 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 8eca434..118572b 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,10 @@ frame = my_function_to_get_audio_frame() prediction = model.predict(frame) ``` +## Reccomended Settings + +While the default settings for openWakeWord will work well in many cases, there are adjustable parameters that can improve performance in some cases. On supported platforms (currently only X86 and Arm64 linux), Speex noise suppression can be enabled by setting the `enable_speex_noise_suppression=True` when instantiating an openWakeWord model. This can improve performance when relatively constant background noise is present. Second, a voice activity deteciton (VAD) model from [Silero](https://github.com/snakers4/silero-vad) is included with openWakeWord, and can be enabled by setting the `vad_threshold` argument to a value between 0 and 1 when instantiating an openWakeWord model. This will only allow a positive prediction from openWakeWord when the VAD model simultaneously has a score above the specified threshold, which can significantly reduce false-positive activations in the present of non-speech noise. Finally, all of the included openWakeWord models were trained to work well with a default threshold of `0.5` for a positive prediction, but you are encouraged to determine the best threshold for your environment and use-case through testing. + # Project Goals openWakeWord has four high-level goals, which combine to (hopefully!) produce a framework that is simple to use *and* extend. @@ -64,6 +68,7 @@ The table below lists each model, examples of the word/phrases it is trained to | ------------- | ------------- | ------------- | | alexa | "alexa"| [docs](docs/models/alexa.md) | | hey mycroft | "hey mycroft" | [docs](docs/models/hey_mycroft.md) | +| hey jarvis | "hey jarvis" | [docs](docs/models/hey_jarvis.md) | | current weather | "what's the weather" | [docs](docs/models/weather.md) | | timers | "set a 10 minute timer" | [docs](docs/models/timers.md) | @@ -157,4 +162,4 @@ Future release roadmaps may have non-english support. In particular, [Mycroft.AI # License -All of the code in openWakeWord is licensed under the **Apache 2.0** license. All of the included pre-trained models are licensed under the [Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-nc-sa/4.0/) license due to the inclusion of datasets with unknown or restrictive licensing as part of the training data. If you are interested in pre-trained models with more permissive licensing, please raise in issue and we will try to add them to a future release. \ No newline at end of file +All of the code in openWakeWord is licensed under the **Apache 2.0** license. All of the included pre-trained models are licensed under the [Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-nc-sa/4.0/) license due to the inclusion of datasets with unknown or restrictive licensing as part of the training data. If you are interested in pre-trained models with more permissive licensing, please raise an issue and we will try to add them to a future release. \ No newline at end of file diff --git a/docs/models/alexa.md b/docs/models/alexa.md index f829944..e976cf6 100755 --- a/docs/models/alexa.md +++ b/docs/models/alexa.md @@ -4,6 +4,37 @@ A model trained to detect the presence of the word "Alexa" in an audio recording Other similar phrases such as "Hey Alexa" or "Alexa stop" may also work, but likely with higher false-reject rates. Similarly, a short pause after the speaking the wakeword is recommended, but the model may also detect the presence of the wakeword is a continuous stream of speech in certain cases. +# Model Architecture + +The model is a simple 3-layer full-connected network, that takes the flattened input features from the frozen audio embedding mode. ReLU activations and layer norms are inserted between the layers. A representive (but not exact) example of this structure is shown below. + +``` +========================================================================================== +Layer (type:depth-idx) Output Shape Param # +========================================================================================== +Sequential [1, 1] -- +├─Flatten: 1-1 [1, 1536] -- +├─Linear: 1-2 [1, 64] 98,368 +├─LayerNorm: 1-3 [1, 64] 128 +├─ReLU: 1-4 [1, 64] -- +├─Linear: 1-5 [1, 64] 4,160 +├─LayerNorm: 1-6 [1, 64] 128 +├─ReLU: 1-7 [1, 64] -- +├─Linear: 1-8 [1, 1] 65 +├─Sigmoid: 1-9 [1, 1] -- +========================================================================================== +Total params: 102,849 +Trainable params: 102,849 +Non-trainable params: 0 +Total mult-adds (M): 0.10 +========================================================================================== +Input size (MB): 0.01 +Forward/backward pass size (MB): 0.00 +Params size (MB): 0.41 +Estimated Total Size (MB): 0.42 +========================================================================================== +``` + # Training Data ## Positive Data diff --git a/docs/models/hey_mycroft.md b/docs/models/hey_mycroft.md index 9e5a0d8..eaf871f 100644 --- a/docs/models/hey_mycroft.md +++ b/docs/models/hey_mycroft.md @@ -4,6 +4,37 @@ A model trained to detect the presence of the phrase "hey mycroft" in an audio r Other similar phrases such as just "mycroft" or may also work, but likely with higher false-reject rates. Similarly, a short pause after the speaking the wakeword is recommended, but the model may also detect the presence of the wakeword is a continuous stream of speech in certain cases. +# Model Architecture + +The model is a simple 3-layer full-connected network, that takes the flattened input features from the frozen audio embedding mode. ReLU activations and layer norms are inserted between the layers. A representive (but not exact) example of this structure is shown below. + +``` +========================================================================================== +Layer (type:depth-idx) Output Shape Param # +========================================================================================== +Sequential [1, 1] -- +├─Flatten: 1-1 [1, 1536] -- +├─Linear: 1-2 [1, 64] 98,368 +├─LayerNorm: 1-3 [1, 64] 128 +├─ReLU: 1-4 [1, 64] -- +├─Linear: 1-5 [1, 64] 4,160 +├─LayerNorm: 1-6 [1, 64] 128 +├─ReLU: 1-7 [1, 64] -- +├─Linear: 1-8 [1, 1] 65 +├─Sigmoid: 1-9 [1, 1] -- +========================================================================================== +Total params: 102,849 +Trainable params: 102,849 +Non-trainable params: 0 +Total mult-adds (M): 0.10 +========================================================================================== +Input size (MB): 0.01 +Forward/backward pass size (MB): 0.00 +Params size (MB): 0.41 +Estimated Total Size (MB): 0.42 +========================================================================================== +``` + # Training Data ## Positive Data diff --git a/docs/models/timers.md b/docs/models/timers.md index ac1c7c3..92a0187 100644 --- a/docs/models/timers.md +++ b/docs/models/timers.md @@ -4,6 +4,35 @@ A model trained to detect the presence of several different phrases all related As with other models, similar phrases beyond those included in the training data may also work, but likely with higher false-reject rates. Similarly, a short pause after the speaking the wake phrase is recommended, but the model may also detect the presence of the wake phrase is a continuous stream of speech in certain cases. +# Model Architecture + +The model is a simple 3-layer full-connected network, that takes the flattened input features from the frozen audio embedding mode. As this model is multi-class, the final layer has the number of nodes equal to the number of classes. A softmax layer is added prior to saving the model to return scores that sum to one across the classes. A representive (but not exact) example of this structure is shown below. + +``` +========================================================================================== +Layer (type:depth-idx) Output Shape Param # +========================================================================================== +Sequential [1, 7] -- +├─Flatten: 1-1 [1, 3264] -- +├─Linear: 1-2 [1, 128] 417,920 +├─ReLU: 1-3 [1, 128] -- +├─Linear: 1-4 [1, 128] 16,512 +├─ReLU: 1-5 [1, 128] -- +├─Linear: 1-6 [1, 7] 903 +├─ReLU: 1-7 [1, 7] -- +========================================================================================== +Total params: 435,335 +Trainable params: 435,335 +Non-trainable params: 0 +Total mult-adds (M): 0.44 +========================================================================================== +Input size (MB): 0.01 +Forward/backward pass size (MB): 0.00 +Params size (MB): 1.74 +Estimated Total Size (MB): 1.76 +========================================================================================== +``` + # Training Data ## Positive Data diff --git a/docs/models/weather.md b/docs/models/weather.md index 2c3d12d..c9feecf 100644 --- a/docs/models/weather.md +++ b/docs/models/weather.md @@ -4,6 +4,38 @@ A model trained to detect the presence of several different phrases all related As with other models, similar phrases beyond those included in the training data may also work, but likely with higher false-reject rates. Similarly, a short pause after the speaking the wake phrase is recommended, but the model may also detect the presence of the wake phrase is a continuous stream of speech in certain cases. +# Model Architecture + +The model is a simple 3-layer full-connected network, that takes the flattened input features from the frozen audio embedding mode. ReLU activations and layer norms are inserted between the layers. A representive (but not exact) example of this structure is shown below. + +``` +========================================================================================== +Layer (type:depth-idx) Output Shape Param # +========================================================================================== +Sequential [1, 1] -- +├─Flatten: 1-1 [1, 1536] -- +├─Linear: 1-2 [1, 64] 98,368 +├─LayerNorm: 1-3 [1, 64] 128 +├─ReLU: 1-4 [1, 64] -- +├─Linear: 1-5 [1, 64] 4,160 +├─LayerNorm: 1-6 [1, 64] 128 +├─ReLU: 1-7 [1, 64] -- +├─Linear: 1-8 [1, 1] 65 +├─Sigmoid: 1-9 [1, 1] -- +========================================================================================== +Total params: 102,849 +Trainable params: 102,849 +Non-trainable params: 0 +Total mult-adds (M): 0.10 +========================================================================================== +Input size (MB): 0.01 +Forward/backward pass size (MB): 0.00 +Params size (MB): 0.41 +Estimated Total Size (MB): 0.42 +========================================================================================== +``` + + # Training Data ## Positive Data diff --git a/openwakeword/metrics.py b/openwakeword/metrics.py index e449b0b..e4a93df 100644 --- a/openwakeword/metrics.py +++ b/openwakeword/metrics.py @@ -40,7 +40,7 @@ def get_false_positives(scores: List, threshold: float, grouping_window: int = 5 n = grouping_window for t in transitions: if bin_pred[t.end()] != 0: - bin_pred[t.end():t.end() + n] = [0]*n + bin_pred[t.end():t.end() + min(len(transitions) - t.end(), n)] = [0]*min(len(transitions) - t.end(), n) return sum(bin_pred) diff --git a/openwakeword/model.py b/openwakeword/model.py index 03986db..ab9fedc 100755 --- a/openwakeword/model.py +++ b/openwakeword/model.py @@ -215,11 +215,12 @@ class Model(): if timing: timing_dict["models"]["vad"] = time.time() - vad_start - # Get frames from last 0.4 to 0.56 seconds (3 frames) and get max VAD score + # Get frames from last 0.4 to 0.56 seconds (3 frames) before the current + # frame and get max VAD score vad_frames = list(self.vad.prediction_buffer)[-7:-4] - vad_avg_score = np.max(vad_frames) if len(vad_frames) > 0 else 0 + vad_max_score = np.max(vad_frames) if len(vad_frames) > 0 else 0 for mdl in predictions.keys(): - if vad_avg_score < self.vad_threshold: + if vad_max_score < self.vad_threshold: predictions[mdl] = 0.0 if timing: diff --git a/openwakeword/resources/models/hey_jarvis_v0.1.onnx b/openwakeword/resources/models/hey_jarvis_v0.1.onnx index b9b51e4..a45f1de 100644 --- a/openwakeword/resources/models/hey_jarvis_v0.1.onnx +++ b/openwakeword/resources/models/hey_jarvis_v0.1.onnx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6ad138060d6426d43f7d30fda4e639329e2751be9ea3cde20294d32a890f8e21 -size 413521 +oid sha256:94a13cfe60075b132f6a472e7e462e8123ee70861bc3fb58434a73712ee0d2cb +size 1271370 diff --git a/openwakeword/utils.py b/openwakeword/utils.py index e7e8101..1b74151 100644 --- a/openwakeword/utils.py +++ b/openwakeword/utils.py @@ -328,7 +328,7 @@ def bulk_predict( prediction_function (str): The name of the method used to predict on the input audio files (default is the `predict_clip` method) ncpu (int): How many processes to create (up to max of available CPUs) - kwargs (dict): Any other keyword arguments to pass to the model prediction function (`predict_clip`) + kwargs (dict): Any other keyword arguments to pass to the model initialization Returns: dict: A dictionary containing the predictions for each file, with the filepath as the key @@ -348,13 +348,14 @@ def bulk_predict( for chunk in chunks: oww = openwakeword.Model( wakeword_model_paths=wakeword_model_paths, + **kwargs ) mdls.append(oww) def f(clips): results = [] for clip in clips: - results.append({clip: getattr(mdls[-1], prediction_function)(clip, **kwargs)}) + results.append({clip: getattr(mdls[-1], prediction_function)(clip)}) q.put(results) ps.append(Process(target=f, args=(chunk,)))