Small changes/bug fixes, starting refactor in preparation for first release

README.md

@@ -19,7 +19,7 @@ More specifically, openWakeWord aims to:
 # Performance and Evaluation
 
 - Mention 0.5/hour false accept rate for near continuous speech (e.g., dinner party corpus)
-- False-reject rate of 5% means that the chanced of missing two activations is only 0.25%. E.g., if a user on average intentially speaks a wake word/phrase 20 times per day, they would expect to have to try two times once per day, and try three times only once every 20 days (assuming the failed activations aren't correlated and the environmental conditions are such that an activation is expected).
+- False-reject rate of 5% means that the chanced of missing two activations is only 0.25%. E.g., if a user on average intentially speaks a wake word/phrase 20 times per day, they would expect to have to try two times once per day, and try three times only once every 20 days (assuming the failed activations aren't correlated and the environmental conditions are such that an activation would otherwise be expected).
 
 
 # Training New Models

examples/detect_from_microphone.py

@@ -28,10 +28,9 @@ mic_stream = sounddevice.InputStream(
 )
 
 # Load openwakeword model(s)
-model_name = "hey_mycroft_v1"
+model_name = "alexa_v5"
 model = Model(
     wakeword_model_paths=[os.path.join("../", "openwakeword", "resources", "models", model_name + ".onnx")],
-    input_sizes=[16]
 )
 
 # Run capture loop, checking for hotwords

openwakeword/data.py

@@ -32,7 +32,6 @@ def stack_clips(audio_data, clip_size=16000*2):
     """
     Takes an input list of 1D arrays (of different lengths), concatenates them together,
     and then extracts clips of a uniform size by dividing the combined array.
-    Also converts the resulting array to 16-bit PCM format.
 
     Args:
         audio_data (List[ndarray]): A list of 1D numpy arrays to combine and stack
@@ -53,10 +52,10 @@ def stack_clips(audio_data, clip_size=16000*2):
             chunk = np.hstack((chunk, np.zeros(clip_size - chunk.shape[0])))
         new_examples.append(chunk)
     
-    # Convert to 16-bit PCM data
-    X = (np.array(new_examples).astype(np.float32)*32767).astype(np.int16)
+    # # Convert to 16-bit PCM data
+    # X = (np.array(new_examples).astype(np.float32)*32767).astype(np.int16)
 
-    return X
+    return np.array(new_examples)
 
 def load_audio_clips(files, clip_size=32000):
     """
@@ -135,7 +134,7 @@ def convert_clips(input_files, output_files, sr=16000, ncpu=1):
     pool.starmap(_convert_clip, [(i,j) for i,j in zip(input_files, output_files)])
 
 
-def filter_audio_paths(target_dirs, min_length_secs, max_length_secs, duration_method="size", filetype=None):
+def filter_audio_paths(target_dirs, min_length_secs, max_length_secs, duration_method="size", glob_filter=None):
     """
     Gets the paths of wav files in flat target directories, automatically filtering
     out files below/above the specified length (in seconds). Assumes that all
@@ -151,8 +150,8 @@ def filter_audio_paths(target_dirs, min_length_secs, max_length_secs, duration_m
         duration_method (str): Whether to use the file size ('size'), or header information ('header')
                                to estimate the duration of the audio file. 'size' is generally
                                much faster, but assumes that all files in the target directory
-                               are the same type, sample rate, and bitrate.
-        filetype (str): The format of the audio files to include (by extension)
+                               are the same type, sample rate, and bitrate. If None, durations are not calculated.
+        glob_filter (str): A pathlib glob filter string to select specific files within the target directory
     
     Returns:
         tuple: A list of strings corresponding to the paths of the wav files that met the length criteria,
@@ -164,12 +163,12 @@ def filter_audio_paths(target_dirs, min_length_secs, max_length_secs, duration_m
     for target_dir in target_dirs:
         sizes = []
         dir_paths = []
-        if filetype:
-            dir_paths = [str(i) for i in Path(target_dir).glob(f"**/*{filetype}")]
+        if glob_filter:
+            dir_paths = [str(i) for i in Path(target_dir).glob(glob_filter)]
             file_paths.extend(dir_paths)
             sizes.extend([os.path.getsize(i) for i in dir_paths])
         else:
-            for i in os.scandir(target_dir):
+            for i in tqdm(os.scandir(target_dir)):
                 dir_paths.append(i.path)
                 file_paths.append(i.path)
                 sizes.append(i.stat().st_size)
@@ -180,8 +179,12 @@ def filter_audio_paths(target_dirs, min_length_secs, max_length_secs, duration_m
         elif duration_method == "header":
             durations.extend([get_clip_duration(i) for i in tqdm(dir_paths)])
 
-    filtered = [(i,j) for i,j in zip(file_paths, durations) if j >= min_length_secs and j <= max_length_secs]
-    return [i[0] for i in filtered], [i[1] for i in filtered]
+    if durations != []:
+        filtered = [(i,j) for i,j in zip(file_paths, durations) if j >= min_length_secs and j <= max_length_secs]
+        return [i[0] for i in filtered], [i[1] for i in filtered]
+    else:
+        return file_paths, []
+    
 
 
 def estimate_clip_duration(audio_files: list, sizes: list):
@@ -246,6 +249,7 @@ def mix_clips_batch(
         snr_high: float=0,
         start_index: List[int]=[],
         rirs: List[str]=[],
+        shuffle: bool=True,
         seed: int=None
     ):
     """
@@ -263,7 +267,8 @@ def mix_clips_batch(
         snr_high (float): The high snr level of the mixing in db
         start_index (List[int]): The starting position (in samples) for the foreground clip to start in the background clip.
         rirs (List[str]): A list of paths to room impulse response functions (RIR) to convolve with the clips to simulate different recording environments.
-                          Applies a single random from the list RIR file to the entire batch. If empty (the default), nothing is done. 
+                          Applies a single random from the list RIR file to the entire batch. If empty (the default), nothing is done.
+        shuffle (bool): Whether to shuffle the foreground clips before mixing (default: True)
         seed (int): A random seed
 
     Returns:
@@ -274,6 +279,9 @@ def mix_clips_batch(
         np.random.seed(seed)
         random.seed(seed)
 
+    if shuffle:
+        random.shuffle(foreground_clips)
+
     # Set start indices, if needed
     if not start_index:
         start_index = [0]*batch_size
@@ -301,17 +309,17 @@ def mix_clips_batch(
             fg_rms, bg_rms = fg.norm(p=2), bg.norm(p=2)
             snr = 10 ** (snr / 20)
             scale = snr * bg_rms / fg_rms
-            start = min(start, combined_size - fg.shape[0])
-            bg[start:start + fg.shape[0]] = bg[start:start + fg.shape[0]] + scale*fg
+            bg[start:start + fg.shape[0]] = bg[start:start + fg.shape[0]] + scale*fg[0:bg.shape[0] - start]
             mixed_clips_batch.append(bg / 2)
             
         mixed_clips_batch = torch.vstack(mixed_clips_batch)
         
         # Apply reverberation to the batch (from a single RIR file)
-        rir_waveform, sr = torchaudio.load(random.choice(rirs))
-        if rir_waveform.shape[0] > 1:
-            rir_waveform = rir_waveform[random.randint(0,rir_waveform.shape[0]-1), :]
-        mixed_clips_batch = reverberate(mixed_clips_batch, rir_waveform, rescale_amp="avg")
+        if rirs:
+            rir_waveform, sr = torchaudio.load(random.choice(rirs))
+            if rir_waveform.shape[0] > 1:
+                rir_waveform = rir_waveform[random.randint(0,rir_waveform.shape[0]-1), :]
+            mixed_clips_batch = reverberate(mixed_clips_batch, rir_waveform, rescale_amp="avg")
 
         # Normalize clips only if max value is outside of [-1, 1]
         abs_max, _ = torch.max(
@@ -324,6 +332,33 @@ def mix_clips_batch(
 
         yield mixed_clips_batch
 
+# Reverberation data augmentation function
+
+def apply_reverb(x, rir_files):
+    """
+    Applies reverberation to the input audio clips
+
+    Args:
+        x (nd.array): A numpy array of shape (batch, audio_samples) containing the audio clips
+        rir_files (Union[str, list]): Either a path to an RIR (room impulse response) file or a list
+                                      of RIR files. If a list, one file will be randomly chosen
+                                      to apply to `x`
+
+    Returns:
+        nd.array: The reverberated audio clips
+    """
+    if isinstance(rir_files, str):
+        rir_waveform, sr = torchaudio.load(rir_files[0])
+    elif isinstance(rir_files, list):
+        rir_waveform, sr = torchaudio.load(random.choice(rir_files))
+
+    # Apply reverberation to the batch (from a single RIR file)
+    if rir_waveform.shape[0] > 1:
+        rir_waveform = rir_waveform[random.randint(0,rir_waveform.shape[0]-1), :]
+    reverbed = reverberate(torch.from_numpy(x), rir_waveform, rescale_amp="avg")
+
+    return reverbed.numpy()
+
 # Load batches of data from mmaped numpy arrays
 class mmap_batch_generator:
     """
@@ -359,8 +394,7 @@ class mmap_batch_generator:
                                     total batch size for each iteration of the generator.
             label_transform_funcs (dict): A dictionary of transformation functions to apply to each batch of labels.
                                           For example, strings can be mapped to integers or one-hot encoded,
-                                          groups of classes can be merged together into one, etc.
-                                    
+                                          groups of classes can be merged together into one, etc.              
         """
         # inputs
         self.data_files = data_files
@@ -368,26 +402,32 @@ class mmap_batch_generator:
         self.data_transform_funcs = data_transform_funcs
         self.label_transform_funcs = label_transform_funcs
         
-        # Get array mmaps and counter object
+        # Get array mmaps and store their shapes
         self.data = {label:np.load(fl, mmap_mode='r') for label, fl in data_files.items()}
         self.data_counter = {label:0 for label in data_files.keys()}
+        self.original_shapes = {label:self.data[label].shape for label in self.data.keys()}
         self.shapes = {label:self.data[label].shape for label in self.data.keys()}
 
-        # Update effective shape of mmpa array based on user-provided transforms
-        for lbl, f in self.data_transform_funcs.items():
-            dummy_data = np.random.random((1, self.shapes[lbl][1], self.shapes[lbl][2]))
-            new_shape = f(dummy_data).shape
-            self.shapes[lbl] = (new_shape[0]*self.shapes[lbl][0], new_shape[1], new_shape[2])
+        # # Update effective shape of mmap array based on user-provided transforms (currently broken)
+        # for lbl, f in self.data_transform_funcs.items():
+        #     dummy_data = np.random.random((1, self.original_shapes[lbl][1], self.original_shapes[lbl][2]))
+        #     new_shape = f(dummy_data).shape
+        #     self.shapes[lbl] = (new_shape[0]*self.original_shapes[lbl][0], new_shape[1], new_shape[2])
 
         # Calculate batch sizes, if the user didn't specify them
         if not self.n_per_class:
             self.n_per_class = {}
             for lbl, shape in self.shapes.items():
+                dummy_data = np.random.random((10, self.shapes[lbl][1], self.shapes[lbl][2]))
+                if self.data_transform_funcs.get(lbl, None):
+                    scale_factor = self.data_transform_funcs.get(lbl, None)(dummy_data).shape[0]/10
+                else:
+                    scale_factor = 1
                 ratio = self.shapes[lbl][0]/sum([i[0] for i in self.shapes.values()])
-                self.n_per_class[lbl] = max(1, int(batch_size*ratio))
+                self.n_per_class[lbl] = max(1, int(int(batch_size*ratio)/scale_factor))
 
         # Get estimated batches per epoch, including the effect of any user-provided transforms
-        batch_size = sum([val for val in self.n_per_class.values()])
+        batch_size = sum([val*scale_factor for val in self.n_per_class.values()])
         batches_per_epoch = sum([i[0] for i in self.shapes.values()])//batch_size
         self.batch_per_epoch = batches_per_epoch
         print("Batches/steps per epoch:", batches_per_epoch)

openwakeword/model.py

@@ -13,18 +13,33 @@
 # limitations under the License.
 
 # Imports
-import onnxruntime as ort
 import numpy as np
+import onnxruntime as ort
+from openwakeword.utils import AudioFeatures
+
+import statistics
 import wave
 import os
-from collections import deque
-from openwakeword.utils import AudioFeatures
+from collections import deque, defaultdict
 from typing import List
-from functools import reduce
+from functools import partial
 import time
+import pprint
+from typing import List, Union
 
 class Model():
-    def __init__(self, wakeword_model_paths: List[str], input_sizes: List[int], **kwargs):
+    """
+    The main model class for openWakeWord. Creates a model object with the shared audio pre-processer
+    and for arbitrarily many custom wake word/wake phrase models.
+    """
+    def __init__(self, wakeword_model_paths: List[str], **kwargs):
+        """
+        Initialize the openWakeWord model object.
+
+        Args:
+            wakeword_model_paths (List[str]): A list of paths of ONNX models to load into the openWakeWord model object
+        """
+
         # Initialize the ONNX models and store them
         sessionOptions = ort.SessionOptions()
         sessionOptions.inter_op_num_threads = 1
@@ -34,19 +49,77 @@ class Model():
         self.models = {}
         self.model_inputs = {}
         self.model_input_names = {}
-        for size, mdl_path in zip(input_sizes, wakeword_model_paths):
+        for mdl_path in wakeword_model_paths:
             mdl_name = mdl_path.split(os.path.sep)[-1].strip(".onnx")
             self.models[mdl_name] = ort.InferenceSession(mdl_path, sess_options=sessionOptions, providers=["CPUExecutionProvider"])
-            self.model_inputs[mdl_name] = size
+            self.model_inputs[mdl_name] = self.models[mdl_name].get_inputs()[0].shape[1]
             self.model_input_names[mdl_name] = self.models[mdl_name].get_inputs()[0].name
 
         # Create buffer to store frame predictios
-        self.prediction_buffer = deque(maxlen=5)
+        self.prediction_buffer = defaultdict(partial(deque, maxlen=5))
 
         # Create AudioFeatures object
         self.preprocessor = AudioFeatures(**kwargs)
 
-    def predict_clip(self, clip, padding=True, **kwargs):
+
+    def predict(self, x: Union[np.ndarray, List], median_smooth: bool=False, timing: bool=False):
+        """Predict with all of the wakeword models on the input audio frames
+        
+        Args:
+            x (Union[ndarray, List]): The input audio data to predict on with the models. Must be 1280 samples of 16khz, 16-bit audio data.
+            median_smooth (bool): Whether to apply a running median smooth of the last three predictions before returning a score.
+                                  Can reduce false-positive productions at the cost of a lower true-positive rate.
+            timing (bool): Whether to print timing information of the models. Can be useful to debug and assess how efficiently models
+                           are running the current hardware.
+
+        Returns:
+            dict: A dictionary of scores between 0 and 1 for each model, where 0 indicates no wake-word/wake-phrase detected
+        """
+        # Get audio features
+        if timing:
+            timing_dict = {}
+            timing_dict["models"] = {}
+            feature_start = time.time()
+        
+        self.preprocessor(x)
+
+        if timing:
+            feature_end = time.time()
+            timing_dict["preprocessor"] = feature_end - feature_start
+
+        # Get predictions from model(s)
+        predictions = {}
+        for mdl in self.models.keys():
+            input_name = self.model_input_names[mdl]
+
+            if timing:
+                model_start = time.time()
+
+            # Run model
+            predictions[mdl] = self.models[mdl].run(
+                                    None,
+                                    {input_name: self.preprocessor.get_features(self.model_inputs[mdl])}
+                                )[0][0][0]
+
+            # Update prediction buffer
+            self.prediction_buffer[mdl].append(predictions[mdl])
+
+            # (Optionally) Smooth model predictions with simple median calculate of last three predictions
+            if median_smooth:
+                predictions[mdl] = statistics.median(list(self.prediction_buffer[mdl])[-3:])
+
+            if timing:
+                model_end = time.time()
+                timing_dict["models"][mdl] = model_end - model_start
+
+        if timing:
+            pp = pprint.PrettyPrinter().pprint(timing_dict)
+            return predictions
+        else:
+            return predictions
+
+
+    def predict_clip(self, clip: str, padding: bool=True, **kwargs):
         """Predict on an full audio clip, simulating streaming prediction.
         The input clip must bit a 16-bit, 16 khz, single-channel WAV file.
 
@@ -72,62 +145,4 @@ class Model():
         for i in range(0, data.shape[0]-step_size, step_size):
             predictions.append(self.predict(data[i:i+step_size], **kwargs))
 
-        return predictions
-
-    def predict(self, x, threshold=0.5, verify_rounds=None, timing=False):
-        """Predict with all of the wakeword models on the input audio frames"""
-        # Get audio features
-        if timing:
-            timing_dict = {}
-            timing_dict["models"] = {}
-            feature_start = time.time()
-        
-        self.preprocessor(x)
-
-        if timing:
-            feature_end = time.time()
-            timing_dict["preprocessor"] = feature_end - feature_start
-
-        # Get predictions from model(s)
-        predictions = {}
-        for mdl in self.models.keys():
-            input_name = self.model_input_names[mdl]
-
-            if timing:
-                model_start = time.time()
-
-            predictions[mdl] = self.models[mdl].run(
-                                    None,
-                                    {input_name: self.preprocessor.get_features(self.model_inputs[mdl])}
-                                )[0][0][0]
-
-            if verify_rounds is not None and predictions[mdl] >= threshold \
-                and len(self.prediction_buffer) > 0 and self.prediction_buffer[-1] < threshold:
-                # Use TTA (test time augmentation) to re-check positive predictions
-                tta_predictions = []
-                offset = 200
-                for round in range(1, verify_rounds+1):
-                    x = list(self.preprocessor.raw_data_buffer)[-32000-offset*round:-offset*round] # arbitrary chunk size, adjust as needed?
-                    tta_predictions.append(self.models[mdl].run(
-                                    None,
-                                    {input_name: self.preprocessor._get_embeddings(x)[None,]}
-                                )[0][0][0])
-                
-                # Update prediction score
-                predictions[mdl] = reduce(lambda x, y: x*y, tta_predictions)
-                self.prediction_buffer.append(predictions[mdl])
-
-            else:
-                # Update prediction buffer
-                self.prediction_buffer.append(predictions[mdl])
-
-            if timing:
-                model_end = time.time()
-                timing_dict["models"][mdl] = model_end - model_start
-
-        if timing:
-            return predictions, timing_dict
-        else:
-            return predictions
-
-        
+        return predictions

openwakeword/utils.py

@@ -44,9 +44,9 @@ class AudioFeatures():
 
         # Create databuffers
         self.raw_data_buffer = deque(maxlen=sr*10)
-        self.melspectrogram_buffer = np.zeros((0,32)) #n_frames x num_features
+        self.melspectrogram_buffer = np.ones((76,32)) #n_frames x num_features
         self.melspectrogram_max_len = 10*97 # 97 is the number of frames in 1 second of 16hz audio
-        self.feature_buffer = np.zeros((32,96))
+        self.feature_buffer = self._get_embeddings(np.zeros(53000).astype(np.int16)) #fill feature buffer with blank data to start
         self.feature_buffer_max_len = 120 # ~10 seconds of feature buffer history
         
     def _get_melspectrogram(self, x, melspec_transform = lambda x: x/10 + 2):