diff --git a/README.md b/README.md index bb10b71..f4d280d 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ More specifically, openWakeWord aims to: # Performance and Evaluation - Mention 0.5/hour false accept rate for near continuous speech (e.g., dinner party corpus) -- False-reject rate of 5% means that the chanced of missing two activations is only 0.25%. E.g., if a user on average intentially speaks a wake word/phrase 20 times per day, they would expect to have to try two times once per day, and try three times only once every 20 days (assuming the failed activations aren't correlated and the environmental conditions are such that an activation is expected). +- False-reject rate of 5% means that the chanced of missing two activations is only 0.25%. E.g., if a user on average intentially speaks a wake word/phrase 20 times per day, they would expect to have to try two times once per day, and try three times only once every 20 days (assuming the failed activations aren't correlated and the environmental conditions are such that an activation would otherwise be expected). # Training New Models diff --git a/examples/detect_from_microphone.py b/examples/detect_from_microphone.py index 9c7513e..eb102fb 100644 --- a/examples/detect_from_microphone.py +++ b/examples/detect_from_microphone.py @@ -28,10 +28,9 @@ mic_stream = sounddevice.InputStream( ) # Load openwakeword model(s) -model_name = "hey_mycroft_v1" +model_name = "alexa_v5" model = Model( wakeword_model_paths=[os.path.join("../", "openwakeword", "resources", "models", model_name + ".onnx")], - input_sizes=[16] ) # Run capture loop, checking for hotwords diff --git a/openwakeword/data.py b/openwakeword/data.py index 3ae6054..f4b5f5d 100644 --- a/openwakeword/data.py +++ b/openwakeword/data.py @@ -32,7 +32,6 @@ def stack_clips(audio_data, clip_size=16000*2): """ Takes an input list of 1D arrays (of different lengths), concatenates them together, and then extracts clips of a uniform size by dividing the combined array. - Also converts the resulting array to 16-bit PCM format. Args: audio_data (List[ndarray]): A list of 1D numpy arrays to combine and stack @@ -53,10 +52,10 @@ def stack_clips(audio_data, clip_size=16000*2): chunk = np.hstack((chunk, np.zeros(clip_size - chunk.shape[0]))) new_examples.append(chunk) - # Convert to 16-bit PCM data - X = (np.array(new_examples).astype(np.float32)*32767).astype(np.int16) + # # Convert to 16-bit PCM data + # X = (np.array(new_examples).astype(np.float32)*32767).astype(np.int16) - return X + return np.array(new_examples) def load_audio_clips(files, clip_size=32000): """ @@ -135,7 +134,7 @@ def convert_clips(input_files, output_files, sr=16000, ncpu=1): pool.starmap(_convert_clip, [(i,j) for i,j in zip(input_files, output_files)]) -def filter_audio_paths(target_dirs, min_length_secs, max_length_secs, duration_method="size", filetype=None): +def filter_audio_paths(target_dirs, min_length_secs, max_length_secs, duration_method="size", glob_filter=None): """ Gets the paths of wav files in flat target directories, automatically filtering out files below/above the specified length (in seconds). Assumes that all @@ -151,8 +150,8 @@ def filter_audio_paths(target_dirs, min_length_secs, max_length_secs, duration_m duration_method (str): Whether to use the file size ('size'), or header information ('header') to estimate the duration of the audio file. 'size' is generally much faster, but assumes that all files in the target directory - are the same type, sample rate, and bitrate. - filetype (str): The format of the audio files to include (by extension) + are the same type, sample rate, and bitrate. If None, durations are not calculated. + glob_filter (str): A pathlib glob filter string to select specific files within the target directory Returns: tuple: A list of strings corresponding to the paths of the wav files that met the length criteria, @@ -164,12 +163,12 @@ def filter_audio_paths(target_dirs, min_length_secs, max_length_secs, duration_m for target_dir in target_dirs: sizes = [] dir_paths = [] - if filetype: - dir_paths = [str(i) for i in Path(target_dir).glob(f"**/*{filetype}")] + if glob_filter: + dir_paths = [str(i) for i in Path(target_dir).glob(glob_filter)] file_paths.extend(dir_paths) sizes.extend([os.path.getsize(i) for i in dir_paths]) else: - for i in os.scandir(target_dir): + for i in tqdm(os.scandir(target_dir)): dir_paths.append(i.path) file_paths.append(i.path) sizes.append(i.stat().st_size) @@ -180,8 +179,12 @@ def filter_audio_paths(target_dirs, min_length_secs, max_length_secs, duration_m elif duration_method == "header": durations.extend([get_clip_duration(i) for i in tqdm(dir_paths)]) - filtered = [(i,j) for i,j in zip(file_paths, durations) if j >= min_length_secs and j <= max_length_secs] - return [i[0] for i in filtered], [i[1] for i in filtered] + if durations != []: + filtered = [(i,j) for i,j in zip(file_paths, durations) if j >= min_length_secs and j <= max_length_secs] + return [i[0] for i in filtered], [i[1] for i in filtered] + else: + return file_paths, [] + def estimate_clip_duration(audio_files: list, sizes: list): @@ -246,6 +249,7 @@ def mix_clips_batch( snr_high: float=0, start_index: List[int]=[], rirs: List[str]=[], + shuffle: bool=True, seed: int=None ): """ @@ -263,7 +267,8 @@ def mix_clips_batch( snr_high (float): The high snr level of the mixing in db start_index (List[int]): The starting position (in samples) for the foreground clip to start in the background clip. rirs (List[str]): A list of paths to room impulse response functions (RIR) to convolve with the clips to simulate different recording environments. - Applies a single random from the list RIR file to the entire batch. If empty (the default), nothing is done. + Applies a single random from the list RIR file to the entire batch. If empty (the default), nothing is done. + shuffle (bool): Whether to shuffle the foreground clips before mixing (default: True) seed (int): A random seed Returns: @@ -274,6 +279,9 @@ def mix_clips_batch( np.random.seed(seed) random.seed(seed) + if shuffle: + random.shuffle(foreground_clips) + # Set start indices, if needed if not start_index: start_index = [0]*batch_size @@ -301,17 +309,17 @@ def mix_clips_batch( fg_rms, bg_rms = fg.norm(p=2), bg.norm(p=2) snr = 10 ** (snr / 20) scale = snr * bg_rms / fg_rms - start = min(start, combined_size - fg.shape[0]) - bg[start:start + fg.shape[0]] = bg[start:start + fg.shape[0]] + scale*fg + bg[start:start + fg.shape[0]] = bg[start:start + fg.shape[0]] + scale*fg[0:bg.shape[0] - start] mixed_clips_batch.append(bg / 2) mixed_clips_batch = torch.vstack(mixed_clips_batch) # Apply reverberation to the batch (from a single RIR file) - rir_waveform, sr = torchaudio.load(random.choice(rirs)) - if rir_waveform.shape[0] > 1: - rir_waveform = rir_waveform[random.randint(0,rir_waveform.shape[0]-1), :] - mixed_clips_batch = reverberate(mixed_clips_batch, rir_waveform, rescale_amp="avg") + if rirs: + rir_waveform, sr = torchaudio.load(random.choice(rirs)) + if rir_waveform.shape[0] > 1: + rir_waveform = rir_waveform[random.randint(0,rir_waveform.shape[0]-1), :] + mixed_clips_batch = reverberate(mixed_clips_batch, rir_waveform, rescale_amp="avg") # Normalize clips only if max value is outside of [-1, 1] abs_max, _ = torch.max( @@ -324,6 +332,33 @@ def mix_clips_batch( yield mixed_clips_batch +# Reverberation data augmentation function + +def apply_reverb(x, rir_files): + """ + Applies reverberation to the input audio clips + + Args: + x (nd.array): A numpy array of shape (batch, audio_samples) containing the audio clips + rir_files (Union[str, list]): Either a path to an RIR (room impulse response) file or a list + of RIR files. If a list, one file will be randomly chosen + to apply to `x` + + Returns: + nd.array: The reverberated audio clips + """ + if isinstance(rir_files, str): + rir_waveform, sr = torchaudio.load(rir_files[0]) + elif isinstance(rir_files, list): + rir_waveform, sr = torchaudio.load(random.choice(rir_files)) + + # Apply reverberation to the batch (from a single RIR file) + if rir_waveform.shape[0] > 1: + rir_waveform = rir_waveform[random.randint(0,rir_waveform.shape[0]-1), :] + reverbed = reverberate(torch.from_numpy(x), rir_waveform, rescale_amp="avg") + + return reverbed.numpy() + # Load batches of data from mmaped numpy arrays class mmap_batch_generator: """ @@ -359,8 +394,7 @@ class mmap_batch_generator: total batch size for each iteration of the generator. label_transform_funcs (dict): A dictionary of transformation functions to apply to each batch of labels. For example, strings can be mapped to integers or one-hot encoded, - groups of classes can be merged together into one, etc. - + groups of classes can be merged together into one, etc. """ # inputs self.data_files = data_files @@ -368,26 +402,32 @@ class mmap_batch_generator: self.data_transform_funcs = data_transform_funcs self.label_transform_funcs = label_transform_funcs - # Get array mmaps and counter object + # Get array mmaps and store their shapes self.data = {label:np.load(fl, mmap_mode='r') for label, fl in data_files.items()} self.data_counter = {label:0 for label in data_files.keys()} + self.original_shapes = {label:self.data[label].shape for label in self.data.keys()} self.shapes = {label:self.data[label].shape for label in self.data.keys()} - # Update effective shape of mmpa array based on user-provided transforms - for lbl, f in self.data_transform_funcs.items(): - dummy_data = np.random.random((1, self.shapes[lbl][1], self.shapes[lbl][2])) - new_shape = f(dummy_data).shape - self.shapes[lbl] = (new_shape[0]*self.shapes[lbl][0], new_shape[1], new_shape[2]) + # # Update effective shape of mmap array based on user-provided transforms (currently broken) + # for lbl, f in self.data_transform_funcs.items(): + # dummy_data = np.random.random((1, self.original_shapes[lbl][1], self.original_shapes[lbl][2])) + # new_shape = f(dummy_data).shape + # self.shapes[lbl] = (new_shape[0]*self.original_shapes[lbl][0], new_shape[1], new_shape[2]) # Calculate batch sizes, if the user didn't specify them if not self.n_per_class: self.n_per_class = {} for lbl, shape in self.shapes.items(): + dummy_data = np.random.random((10, self.shapes[lbl][1], self.shapes[lbl][2])) + if self.data_transform_funcs.get(lbl, None): + scale_factor = self.data_transform_funcs.get(lbl, None)(dummy_data).shape[0]/10 + else: + scale_factor = 1 ratio = self.shapes[lbl][0]/sum([i[0] for i in self.shapes.values()]) - self.n_per_class[lbl] = max(1, int(batch_size*ratio)) + self.n_per_class[lbl] = max(1, int(int(batch_size*ratio)/scale_factor)) # Get estimated batches per epoch, including the effect of any user-provided transforms - batch_size = sum([val for val in self.n_per_class.values()]) + batch_size = sum([val*scale_factor for val in self.n_per_class.values()]) batches_per_epoch = sum([i[0] for i in self.shapes.values()])//batch_size self.batch_per_epoch = batches_per_epoch print("Batches/steps per epoch:", batches_per_epoch) diff --git a/openwakeword/model.py b/openwakeword/model.py index e03805b..5e2c0a1 100644 --- a/openwakeword/model.py +++ b/openwakeword/model.py @@ -13,18 +13,33 @@ # limitations under the License. # Imports -import onnxruntime as ort import numpy as np +import onnxruntime as ort +from openwakeword.utils import AudioFeatures + +import statistics import wave import os -from collections import deque -from openwakeword.utils import AudioFeatures +from collections import deque, defaultdict from typing import List -from functools import reduce +from functools import partial import time +import pprint +from typing import List, Union class Model(): - def __init__(self, wakeword_model_paths: List[str], input_sizes: List[int], **kwargs): + """ + The main model class for openWakeWord. Creates a model object with the shared audio pre-processer + and for arbitrarily many custom wake word/wake phrase models. + """ + def __init__(self, wakeword_model_paths: List[str], **kwargs): + """ + Initialize the openWakeWord model object. + + Args: + wakeword_model_paths (List[str]): A list of paths of ONNX models to load into the openWakeWord model object + """ + # Initialize the ONNX models and store them sessionOptions = ort.SessionOptions() sessionOptions.inter_op_num_threads = 1 @@ -34,19 +49,77 @@ class Model(): self.models = {} self.model_inputs = {} self.model_input_names = {} - for size, mdl_path in zip(input_sizes, wakeword_model_paths): + for mdl_path in wakeword_model_paths: mdl_name = mdl_path.split(os.path.sep)[-1].strip(".onnx") self.models[mdl_name] = ort.InferenceSession(mdl_path, sess_options=sessionOptions, providers=["CPUExecutionProvider"]) - self.model_inputs[mdl_name] = size + self.model_inputs[mdl_name] = self.models[mdl_name].get_inputs()[0].shape[1] self.model_input_names[mdl_name] = self.models[mdl_name].get_inputs()[0].name # Create buffer to store frame predictios - self.prediction_buffer = deque(maxlen=5) + self.prediction_buffer = defaultdict(partial(deque, maxlen=5)) # Create AudioFeatures object self.preprocessor = AudioFeatures(**kwargs) - def predict_clip(self, clip, padding=True, **kwargs): + + def predict(self, x: Union[np.ndarray, List], median_smooth: bool=False, timing: bool=False): + """Predict with all of the wakeword models on the input audio frames + + Args: + x (Union[ndarray, List]): The input audio data to predict on with the models. Must be 1280 samples of 16khz, 16-bit audio data. + median_smooth (bool): Whether to apply a running median smooth of the last three predictions before returning a score. + Can reduce false-positive productions at the cost of a lower true-positive rate. + timing (bool): Whether to print timing information of the models. Can be useful to debug and assess how efficiently models + are running the current hardware. + + Returns: + dict: A dictionary of scores between 0 and 1 for each model, where 0 indicates no wake-word/wake-phrase detected + """ + # Get audio features + if timing: + timing_dict = {} + timing_dict["models"] = {} + feature_start = time.time() + + self.preprocessor(x) + + if timing: + feature_end = time.time() + timing_dict["preprocessor"] = feature_end - feature_start + + # Get predictions from model(s) + predictions = {} + for mdl in self.models.keys(): + input_name = self.model_input_names[mdl] + + if timing: + model_start = time.time() + + # Run model + predictions[mdl] = self.models[mdl].run( + None, + {input_name: self.preprocessor.get_features(self.model_inputs[mdl])} + )[0][0][0] + + # Update prediction buffer + self.prediction_buffer[mdl].append(predictions[mdl]) + + # (Optionally) Smooth model predictions with simple median calculate of last three predictions + if median_smooth: + predictions[mdl] = statistics.median(list(self.prediction_buffer[mdl])[-3:]) + + if timing: + model_end = time.time() + timing_dict["models"][mdl] = model_end - model_start + + if timing: + pp = pprint.PrettyPrinter().pprint(timing_dict) + return predictions + else: + return predictions + + + def predict_clip(self, clip: str, padding: bool=True, **kwargs): """Predict on an full audio clip, simulating streaming prediction. The input clip must bit a 16-bit, 16 khz, single-channel WAV file. @@ -72,62 +145,4 @@ class Model(): for i in range(0, data.shape[0]-step_size, step_size): predictions.append(self.predict(data[i:i+step_size], **kwargs)) - return predictions - - def predict(self, x, threshold=0.5, verify_rounds=None, timing=False): - """Predict with all of the wakeword models on the input audio frames""" - # Get audio features - if timing: - timing_dict = {} - timing_dict["models"] = {} - feature_start = time.time() - - self.preprocessor(x) - - if timing: - feature_end = time.time() - timing_dict["preprocessor"] = feature_end - feature_start - - # Get predictions from model(s) - predictions = {} - for mdl in self.models.keys(): - input_name = self.model_input_names[mdl] - - if timing: - model_start = time.time() - - predictions[mdl] = self.models[mdl].run( - None, - {input_name: self.preprocessor.get_features(self.model_inputs[mdl])} - )[0][0][0] - - if verify_rounds is not None and predictions[mdl] >= threshold \ - and len(self.prediction_buffer) > 0 and self.prediction_buffer[-1] < threshold: - # Use TTA (test time augmentation) to re-check positive predictions - tta_predictions = [] - offset = 200 - for round in range(1, verify_rounds+1): - x = list(self.preprocessor.raw_data_buffer)[-32000-offset*round:-offset*round] # arbitrary chunk size, adjust as needed? - tta_predictions.append(self.models[mdl].run( - None, - {input_name: self.preprocessor._get_embeddings(x)[None,]} - )[0][0][0]) - - # Update prediction score - predictions[mdl] = reduce(lambda x, y: x*y, tta_predictions) - self.prediction_buffer.append(predictions[mdl]) - - else: - # Update prediction buffer - self.prediction_buffer.append(predictions[mdl]) - - if timing: - model_end = time.time() - timing_dict["models"][mdl] = model_end - model_start - - if timing: - return predictions, timing_dict - else: - return predictions - - \ No newline at end of file + return predictions \ No newline at end of file diff --git a/openwakeword/utils.py b/openwakeword/utils.py index c949fd1..8db5a9d 100644 --- a/openwakeword/utils.py +++ b/openwakeword/utils.py @@ -44,9 +44,9 @@ class AudioFeatures(): # Create databuffers self.raw_data_buffer = deque(maxlen=sr*10) - self.melspectrogram_buffer = np.zeros((0,32)) #n_frames x num_features + self.melspectrogram_buffer = np.ones((76,32)) #n_frames x num_features self.melspectrogram_max_len = 10*97 # 97 is the number of frames in 1 second of 16hz audio - self.feature_buffer = np.zeros((32,96)) + self.feature_buffer = self._get_embeddings(np.zeros(53000).astype(np.int16)) #fill feature buffer with blank data to start self.feature_buffer_max_len = 120 # ~10 seconds of feature buffer history def _get_melspectrogram(self, x, melspec_transform = lambda x: x/10 + 2):