From f3e74cd8c4cb9b144ba6fa60f0b6fefb6bf9a352 Mon Sep 17 00:00:00 2001 From: dscripka Date: Mon, 4 Sep 2023 11:15:50 -0400 Subject: [PATCH] Working example of automatic model training complete [skip ci] --- examples/custom_model.yml | 6 +- notebooks/automatic_model_training.ipynb | 342 ++++++++++++++++++++--- openwakeword/data.py | 1 - openwakeword/train.py | 164 ++++++----- 4 files changed, 397 insertions(+), 116 deletions(-) diff --git a/examples/custom_model.yml b/examples/custom_model.yml index 4b5c999..50e0747 100644 --- a/examples/custom_model.yml +++ b/examples/custom_model.yml @@ -69,7 +69,7 @@ feature_data_files: "ACAV100M_sample": "./openwakeword_features_ACAV100M_2000_hrs_16bit.npy" # Define the number of examples from each data file per batch. Note that the key names here -# must correspond to those define in the `negative_data_files` dictionary above (except for +# must correspond to those define in the `feature_data_files` dictionary above (except for # the `positive` and `adversarial_negative` keys, which are automatically defined). The sum # of the values for each key define the total batch size for training. Initial testing indicates # that batch sizes of 1024-4096 work will in practice. @@ -88,7 +88,9 @@ layer_size: 32 # Define training parameters. The values below are recommended defaults for most applications, # but unique deployment environments will likely require testing to determine which values -# are the most appropriate. +# are the most appropriate. Note that all "target_" values are determined from the validation data, +# and since early-stopping is utilized, the final performance of the trained model +# may be slighly overfit to the validation data. steps: 100000 # the maximum number of steps when training the model max_negative_weight: 1500 # the maximum weight to give negative samples during training to reduce false positives diff --git a/notebooks/automatic_model_training.ipynb b/notebooks/automatic_model_training.ipynb index b7e4e02..2827e36 100644 --- a/notebooks/automatic_model_training.ipynb +++ b/notebooks/automatic_model_training.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "4a8bbcb8", + "id": "7790ccc3", "metadata": {}, "source": [ "# Introduction" @@ -10,17 +10,26 @@ }, { "cell_type": "markdown", - "id": "ddd29870", + "id": "3af2f31f", "metadata": {}, "source": [ "This notebook demonstrates how to train custom openWakeWord models using pre-defined datasets and an automated process for dataset generation and training. While not guaranteed to always produce the best performing model, the methods shown in this notebook often produce baseline models with releatively strong performance.\n", "\n", - "Manual data preparation and model training (e.g., see the [training models](training_models.ipynb) notebook) remains an option for when full control over the model development process is needed." + "Manual data preparation and model training (e.g., see the [training models](training_models.ipynb) notebook) remains an option for when full control over the model development process is needed.\n", + "\n", + "At a high level, the automatic training process takes advantages of several techniques to try and produce a good model, including:\n", + "\n", + "- Early-stopping and checkpoint averaging (similar to [stochastic weight averaging](https://arxiv.org/abs/1803.05407) to search for the best models found during training, according to the validation data\n", + "- Variable learning rates with cosine decay and multiple cycles\n", + "- Adaptive batch construction to focus on only high-loss examples when the model begins to converge, combined with gradient accumulation to ensure that batch sizes are still large enough for stable training\n", + "- Cycical weight schedules for negative examples to help the model reduce false-positive rates\n", + "\n", + "See the contents of the `train.py` file for more details." ] }, { "cell_type": "markdown", - "id": "75cebb7c", + "id": "6161cbc6", "metadata": {}, "source": [ "# Environment Setup" @@ -28,7 +37,7 @@ }, { "cell_type": "markdown", - "id": "c9bd1f49", + "id": "fffee4d4", "metadata": {}, "source": [ "To begin, we'll need to install the requirements for training custom models. In particular, a relatively recent version of Pytorch and custom fork of the [piper-sample-generator](https://github.com/dscripka/piper-sample-generator) library for generating synthetic examples for the custom model." @@ -37,7 +46,7 @@ { "cell_type": "code", "execution_count": null, - "id": "645f5330", + "id": "2eed3f89", "metadata": {}, "outputs": [], "source": [ @@ -48,20 +57,31 @@ "!wget -O models/en-us-libritts-high.pt 'https://github.com/rhasspy/piper-sample-generator/releases/download/v1.0.0/en-us-libritts-high.pt'\n", "\n", "# install openwakeword (full installation to support training)\n", - "!pip install openwakeword[full]\n" + "!pip install openwakeword[full]\n", + "!git clone https://github.com/dscripka/openwakeword\n", + "!cd openwakeword\n" ] }, { "cell_type": "code", - "execution_count": 17, - "id": "259e6491", + "execution_count": 1, + "id": "7f556e37", "metadata": { "ExecuteTime": { - "end_time": "2023-09-04T02:00:48.344884Z", - "start_time": "2023-09-04T02:00:48.340514Z" + "end_time": "2023-09-04T13:42:01.183840Z", + "start_time": "2023-09-04T13:41:59.752153Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/dscripka/anaconda3/envs/openwakeword_dev/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ "# Imports\n", "\n", @@ -72,16 +92,12 @@ "import sys\n", "from pathlib import Path\n", "import uuid\n", - "import yaml\n", - "\n", - "# Set paths for locally installed piper-sample-generator\n", - "sys.path.insert(0, \"../../piper-sample-generator/\")\n", - "from generate_samples import generate_samples\n" + "import yaml\n" ] }, { "cell_type": "markdown", - "id": "cb69d8e4", + "id": "0e38803b", "metadata": {}, "source": [ "# Download Data" @@ -89,7 +105,7 @@ }, { "cell_type": "markdown", - "id": "ec4434c8", + "id": "beef6917", "metadata": {}, "source": [ "When training new openWakeWord models using the automated procedure, four specific types of data are required:\n", @@ -110,7 +126,7 @@ { "cell_type": "code", "execution_count": 15, - "id": "4ed7bacd", + "id": "9929f1b7", "metadata": { "ExecuteTime": { "end_time": "2023-09-04T01:07:17.746749Z", @@ -136,7 +152,7 @@ { "cell_type": "code", "execution_count": null, - "id": "4532caf0", + "id": "16becf16", "metadata": {}, "outputs": [], "source": [ @@ -179,7 +195,7 @@ { "cell_type": "code", "execution_count": null, - "id": "3a475459", + "id": "66d26414", "metadata": {}, "outputs": [], "source": [ @@ -194,7 +210,7 @@ }, { "cell_type": "markdown", - "id": "bda8e47e", + "id": "ed004cb1", "metadata": {}, "source": [ "# Define Training Configuration" @@ -202,7 +218,7 @@ }, { "cell_type": "markdown", - "id": "8c2013a4", + "id": "e9385f81", "metadata": {}, "source": [ "For automated model training openWakeWord uses a specially designed training script and a [YAML](https://yaml.org/) configuration file that defines all of the information required for training a new wake word/phrase detection model.\n", @@ -212,17 +228,17 @@ "- We'll train a detection model for the phrase \"hey sebastian\"\n", "- We'll only generate 5,000 positive and negative examples (to save on time for this example)\n", "- We'll only generate 1,000 validation positive and negative examples for early stopping (again to save time)\n", - "- The model will be trained for 30,000 steps (larger datasets will benefit from longer training)\n" + "- The model will only be trained for 10,000 steps (larger datasets will benefit from longer training)\n" ] }, { "cell_type": "code", - "execution_count": 20, - "id": "81bc1bea", + "execution_count": 3, + "id": "87d35ce8", "metadata": { "ExecuteTime": { - "end_time": "2023-09-04T02:03:46.688266Z", - "start_time": "2023-09-04T02:03:46.672580Z" + "end_time": "2023-09-04T13:42:07.998260Z", + "start_time": "2023-09-04T13:42:07.982635Z" } }, "outputs": [ @@ -256,7 +272,7 @@ " 'target_false_positives_per_hour': 0.2}" ] }, - "execution_count": 20, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -269,12 +285,12 @@ }, { "cell_type": "code", - "execution_count": 21, - "id": "d0af4242", + "execution_count": 86, + "id": "fc0fe116", "metadata": { "ExecuteTime": { - "end_time": "2023-09-04T02:30:24.194893Z", - "start_time": "2023-09-04T02:30:24.176938Z" + "end_time": "2023-09-04T15:07:00.859210Z", + "start_time": "2023-09-04T15:07:00.841472Z" } }, "outputs": [], @@ -282,9 +298,21 @@ "# Modify values in the config and save a new version\n", "\n", "config[\"target_phrase\"] = [\"hey sebastian\"]\n", - "config[\"n_samples\"] = 5000\n", + "config[\"n_samples\"] = 1000\n", "config[\"n_samples_val\"] = 1000\n", - "config[\"steps\"] = 30000\n", + "config[\"steps\"] = 10000\n", + "\n", + "## temporary\n", + "config[\"target_accuracy\"] = 0.3\n", + "config[\"target_recall\"] = 0.2\n", + "config[\"piper_sample_generator_path\"] = os.path.abspath(\"../../piper-sample-generator/\")\n", + "config[\"rir_paths\"] = [\"/home/dscripka/dscripkaDrive/Home/computersAndTechnology/machine_learning/speech/openWakeWord_example_data/mit_rirs/16khz\"]\n", + "config[\"background_paths\"] = [\n", + " \"/home/dscripka/dscripkaDrive/Home/computersAndTechnology/machine_learning/speech/openWakeWord_example_data/fsd50k_sample/\",\n", + " \"/home/dscripka/dscripkaDrive/Home/computersAndTechnology/machine_learning/speech/openWakeWord_example_data/fma_sample/\",\n", + "]\n", + "config[\"false_positive_validation_data_path\"] = \"/home/dscripka/dscripkaDrive_nvme_fast/experiments/speech/openWakeWord/wakeword_testing_data/val_set_features.npy\"\n", + "config[\"feature_data_files\"] = {\"ACAV100M_sample\": \"/home/dscripka/dscripkaDrive_nvme_fast/experiments/speech/openWakeWord/wakeword_training_data/negative_examples/openwakeword_features/openwakeword_features_ACAV100M_2000_hrs_16bit.npy\"}\n", "\n", "with open('my_model.yaml', 'w') as file:\n", " documents = yaml.dump(config, file)" @@ -292,15 +320,15 @@ }, { "cell_type": "markdown", - "id": "db52159f", + "id": "676f1caa", "metadata": {}, "source": [ - "# Start Model Training" + "# Train the Model" ] }, { "cell_type": "markdown", - "id": "3e8d66c5", + "id": "77e5b1e0", "metadata": {}, "source": [ "With the data downloaded and training configuration set, we can now start training the model. We'll do this in parts to better illustrate the sequence, but you can also execute every step sequentially for a fully automated process." @@ -308,13 +336,239 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "9dff83ec", - "metadata": {}, - "outputs": [], + "execution_count": 15, + "id": "58eb1bfb", + "metadata": { + "ExecuteTime": { + "end_time": "2023-09-04T13:50:08.803326Z", + "start_time": "2023-09-04T13:50:06.790241Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:root:Generating positive clips for training\r\n", + "WARNING:root:Skipping generation of positive clips for training, as ~1000 already exist\r\n", + "INFO:root:Generating positive clips for testing\r\n", + "WARNING:root:Skipping generation of positive clips testing, as ~1000 already exist\r\n", + "INFO:root:Generating negative clips for training\r\n", + "WARNING:root:Skipping generation of negative clips for training, as ~1000 already exist\r\n", + "INFO:root:Generating negative clips for testing\r\n", + "WARNING:root:Skipping generation of negative clips for testing, as ~1000 already exist\r\n", + "\u001b[0m" + ] + } + ], "source": [ "# Step 1: Generate synthetic clips\n", - "\n" + "# For the number of clips we are using, this should take ~10 minutes on a free Google Colab instance\n", + "# If generation fails, you can simply run this command again as it will continue generating until the\n", + "# number of files meets the targets specified in the config file\n", + "\n", + "!{sys.executable} ../openwakeword/train.py --training_config my_model.yaml --generate_clips" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "7e2fda2e", + "metadata": { + "ExecuteTime": { + "end_time": "2023-09-04T13:56:08.781018Z", + "start_time": "2023-09-04T13:55:40.203515Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:root:Creating augmentation generators\n", + "INFO:root:Computing openwakeword features for generated samples\n", + "Computing features: 100%|███████████████████████| 62/62 [00:05<00:00, 10.58it/s]\n", + "Trimming empty rows: 1it [00:00, 7.99it/s]\n", + "Computing features: 100%|███████████████████████| 62/62 [00:05<00:00, 10.45it/s]\n", + "Trimming empty rows: 1it [00:00, 8.07it/s]\n", + "Computing features: 100%|███████████████████████| 62/62 [00:05<00:00, 10.77it/s]\n", + "Trimming empty rows: 1it [00:00, 8.09it/s]\n", + "Computing features: 100%|███████████████████████| 62/62 [00:05<00:00, 10.53it/s]\n", + "Trimming empty rows: 1it [00:00, 8.07it/s]\n", + "\u001b[0m" + ] + } + ], + "source": [ + "# Step 2: Augment the generated clips\n", + "\n", + "!{sys.executable} ../openwakeword/train.py --training_config my_model.yaml --augment_clips" + ] + }, + { + "cell_type": "code", + "execution_count": 87, + "id": "1a9fafe4", + "metadata": { + "ExecuteTime": { + "end_time": "2023-09-04T15:11:14.742260Z", + "start_time": "2023-09-04T15:07:03.755159Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:root:Starting training sequence 1...\n", + "Training: 75%|████████████████████▏ | 7496/10000 [00:45<00:14, 168.37it/s]1.3274336 0.66 0.331\n", + "INFO:root:Saving checkpoint with metrics >= to targets!\n", + "Training: 76%|█████████████████████▎ | 7621/10000 [00:49<00:28, 82.33it/s]1.3274336 0.66 0.331\n", + "INFO:root:Saving checkpoint with metrics >= to targets!\n", + "Training: 78%|█████████████████████▋ | 7762/10000 [00:53<00:22, 97.84it/s]2.300885 0.6735 0.362\n", + "Training: 79%|██████████████████████ | 7887/10000 [00:57<00:25, 83.36it/s]0.088495575 0.607 0.216\n", + "INFO:root:Saving checkpoint with metrics >= to targets!\n", + "Training: 80%|██████████████████████▍ | 8011/10000 [01:00<00:23, 83.46it/s]2.2123895 0.676 0.367\n", + "Training: 82%|██████████████████████▊ | 8153/10000 [01:04<00:19, 96.94it/s]0.26548672 0.623 0.252\n", + "Training: 83%|███████████████████████▏ | 8277/10000 [01:08<00:20, 82.58it/s]0.088495575 0.605 0.212\n", + "INFO:root:Saving checkpoint with metrics >= to targets!\n", + "Training: 84%|███████████████████████▌ | 8416/10000 [01:12<00:16, 95.46it/s]0.44247788 0.6405 0.29\n", + "Training: 85%|███████████████████████▉ | 8541/10000 [01:16<00:17, 82.74it/s]1.5044248 0.6605 0.335\n", + "Training: 87%|████████████████████████▎ | 8683/10000 [01:19<00:13, 96.74it/s]1.2389381 0.6595 0.333\n", + "Training: 88%|████████████████████████▋ | 8807/10000 [01:23<00:14, 82.40it/s]0.44247788 0.638 0.284\n", + "Training: 89%|█████████████████████████ | 8931/10000 [01:27<00:12, 82.28it/s]0.44247788 0.643 0.295\n", + "Training: 91%|█████████████████████████▍ | 9073/10000 [01:31<00:09, 95.54it/s]0.9734513 0.658 0.327\n", + "Training: 92%|█████████████████████████▊ | 9197/10000 [01:34<00:09, 83.55it/s]1.5929203 0.6635 0.341\n", + "Training: 93%|██████████████████████████▏ | 9338/10000 [01:38<00:06, 96.77it/s]2.0353982 0.6685 0.352\n", + "Training: 95%|██████████████████████████▍ | 9463/10000 [01:42<00:06, 82.58it/s]1.1504425 0.658 0.33\n", + "Training: 96%|██████████████████████████▉ | 9605/10000 [01:46<00:04, 97.06it/s]1.2389381 0.6585 0.331\n", + "Training: 97%|███████████████████████████▏| 9727/10000 [01:49<00:03, 81.79it/s]1.2389381 0.6585 0.331\n", + "Training: 99%|███████████████████████████▌| 9866/10000 [01:53<00:01, 95.62it/s]1.1504425 0.6585 0.33\n", + "Training: 100%|███████████████████████████▉| 9999/10000 [01:57<00:00, 85.19it/s]\n", + "INFO:root:Starting training sequence 2...\n", + "Training: 4%|█▏ | 41/1000.0 [00:00<00:11, 86.52it/s]1.1504425 0.658 0.33\n", + "Training: 9%|██▌ | 89/1000.0 [00:04<00:37, 24.32it/s]1.4159292 0.6595 0.333\n", + "Training: 14%|████ | 143/1000.0 [00:07<00:36, 23.68it/s]1.3274336 0.6585 0.331\n", + "Training: 20%|█████▍ | 195/1000.0 [00:10<00:34, 23.10it/s]1.5044248 0.6605 0.335\n", + "Training: 25%|██████▉ | 247/1000.0 [00:14<00:32, 22.84it/s]1.6814159 0.665 0.344\n", + "Training: 30%|████████▍ | 300/1000.0 [00:17<00:30, 22.77it/s]1.858407 0.668 0.352\n", + "Training: 35%|█████████▉ | 353/1000.0 [00:20<00:27, 23.39it/s]2.1238937 0.671 0.359\n", + "Training: 40%|███████████▎ | 405/1000.0 [00:24<00:27, 22.00it/s]1.858407 0.6685 0.352\n", + "Training: 46%|████████████▊ | 458/1000.0 [00:27<00:23, 23.08it/s]1.4159292 0.6605 0.335\n", + "Training: 51%|██████████████▎ | 512/1000.0 [00:30<00:20, 23.52it/s]1.5929203 0.664 0.342\n", + "Training: 56%|███████████████▊ | 563/1000.0 [00:33<00:19, 22.59it/s]1.1504425 0.659 0.33\n", + "Training: 62%|█████████████████▏ | 616/1000.0 [00:37<00:17, 22.47it/s]0.44247788 0.641 0.291\n", + "Training: 67%|██████████████████▋ | 667/1000.0 [00:40<00:14, 22.93it/s]0.44247788 0.6415 0.292\n", + "Training: 74%|████████████████████▋ | 737/1000.0 [00:43<00:08, 30.03it/s]0.44247788 0.6435 0.296\n", + "Training: 79%|██████████████████████ | 787/1000.0 [00:47<00:08, 23.88it/s]0.53097343 0.646 0.301\n", + "Training: 84%|███████████████████████▍ | 837/1000.0 [00:50<00:07, 22.92it/s]0.53097343 0.646 0.301\n", + "Training: 89%|████████████████████████▊ | 886/1000.0 [00:53<00:05, 22.13it/s]0.53097343 0.646 0.301\n", + "Training: 94%|██████████████████████████▏ | 935/1000.0 [00:57<00:02, 22.19it/s]0.53097343 0.6465 0.303\n", + "Training: 100%|███████████████████████████▉| 999/1000.0 [01:00<00:00, 16.52it/s]\n", + "INFO:root:Starting training sequence 3...\n", + "Training: 5%|█▎ | 47/1000.0 [00:00<00:09, 97.00it/s]0.53097343 0.6465 0.303\n", + "Training: 10%|██▊ | 95/1000.0 [00:04<00:35, 25.72it/s]0.53097343 0.6465 0.303\n", + "Training: 14%|████ | 145/1000.0 [00:07<00:36, 23.53it/s]0.53097343 0.6465 0.303\n", + "Training: 20%|█████▍ | 195/1000.0 [00:10<00:35, 22.67it/s]0.53097343 0.6465 0.303\n", + "Training: 25%|██████▉ | 247/1000.0 [00:14<00:34, 22.09it/s]0.53097343 0.647 0.304\n", + "Training: 30%|████████▎ | 299/1000.0 [00:17<00:30, 22.78it/s]0.7079646 0.6485 0.308\n", + "Training: 35%|█████████▉ | 353/1000.0 [00:20<00:28, 22.88it/s]0.7079646 0.6485 0.308\n", + "Training: 41%|███████████▎ | 406/1000.0 [00:23<00:25, 23.08it/s]0.7079646 0.649 0.309\n", + "Training: 46%|████████████▊ | 457/1000.0 [00:27<00:23, 22.67it/s]0.7079646 0.6495 0.31\n", + "Training: 51%|██████████████▎ | 511/1000.0 [00:30<00:21, 23.20it/s]0.7079646 0.65 0.311\n", + "Training: 56%|███████████████▊ | 563/1000.0 [00:33<00:19, 22.79it/s]0.7079646 0.651 0.313\n", + "Training: 62%|█████████████████▏ | 616/1000.0 [00:37<00:16, 23.22it/s]0.7079646 0.652 0.315\n", + "Training: 67%|██████████████████▋ | 668/1000.0 [00:40<00:14, 22.49it/s]0.7079646 0.653 0.317\n", + "Training: 72%|████████████████████▏ | 720/1000.0 [00:43<00:12, 22.51it/s]0.7079646 0.654 0.319\n", + "Training: 77%|█████████████████████▋ | 774/1000.0 [00:47<00:09, 22.95it/s]0.7079646 0.654 0.319\n", + "Training: 82%|███████████████████████ | 825/1000.0 [00:50<00:07, 22.49it/s]0.7079646 0.6535 0.318\n", + "Training: 88%|████████████████████████▌ | 879/1000.0 [00:53<00:05, 23.56it/s]0.7079646 0.654 0.319\n", + "Training: 95%|██████████████████████████▌ | 947/1000.0 [00:56<00:01, 30.09it/s]0.7079646 0.654 0.319\n", + "Training: 100%|███████████████████████████▉| 999/1000.0 [01:00<00:00, 16.55it/s]\n", + "INFO:root:Merging best checkpoints into single model...\n", + "\n", + "\n", + "INFO:root:\n", + "################\n", + "Final Model Accuracy: 0.637499988079071\n", + "Final Model Recall: 0.2809999883174896\n", + "Final Model False Positives per Hour: 0.44247788190841675\n", + "################\n", + "\n", + "DEBUG:tensorflow:Falling back to TensorFlow client; we recommended you install the Cloud TPU client directly with pip install cloud-tpu-client.\n", + "DEBUG:h5py._conv:Creating converter from 7 to 5\n", + "DEBUG:h5py._conv:Creating converter from 5 to 7\n", + "DEBUG:h5py._conv:Creating converter from 7 to 5\n", + "DEBUG:h5py._conv:Creating converter from 5 to 7\n", + "2023-09-04 11:11:12.446476: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2023-09-04 11:11:12.446795: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcusolver.so.11'; dlerror: libcusolver.so.11: cannot open shared object file: No such file or directory\n", + "2023-09-04 11:11:12.446847: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1850] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.\n", + "Skipping registering GPU devices...\n", + "2023-09-04 11:11:12.446979: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", + "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", + "WARNING:absl:Function `__call__` contains input name(s) onnx_tf__tf_Flatten_0_45bfc89f with unsupported characters which will be renamed to onnx_tf__tf_flatten_0_45bfc89f in the SavedModel.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-09-04 11:11:13.859582: W tensorflow/python/util/util.cc:368] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.\n", + "WARNING:absl:Found untraced functions such as gen_tensor_dict while saving (showing 1 of 1). These functions will not be directly callable after loading.\n", + "INFO:tensorflow:Assets written to: /tmp/tmpqvff79vt/tf_model/assets\n", + "2023-09-04 11:11:14.012590: W tensorflow/compiler/mlir/lite/python/tf_tfl_flatbuffer_helpers.cc:357] Ignored output_format.\n", + "2023-09-04 11:11:14.012606: W tensorflow/compiler/mlir/lite/python/tf_tfl_flatbuffer_helpers.cc:360] Ignored drop_control_dependency.\n", + "2023-09-04 11:11:14.013126: I tensorflow/cc/saved_model/reader.cc:43] Reading SavedModel from: /tmp/tmpqvff79vt/tf_model\n", + "2023-09-04 11:11:14.013536: I tensorflow/cc/saved_model/reader.cc:78] Reading meta graph with tags { serve }\n", + "2023-09-04 11:11:14.013546: I tensorflow/cc/saved_model/reader.cc:119] Reading SavedModel debug info (if present) from: /tmp/tmpqvff79vt/tf_model\n", + "2023-09-04 11:11:14.014464: I tensorflow/cc/saved_model/loader.cc:228] Restoring SavedModel bundle.\n", + "2023-09-04 11:11:14.025569: I tensorflow/cc/saved_model/loader.cc:212] Running initialization op on SavedModel bundle at path: /tmp/tmpqvff79vt/tf_model\n", + "2023-09-04 11:11:14.030851: I tensorflow/cc/saved_model/loader.cc:301] SavedModel load for tags { serve }; Status: success: OK. Took 17727 microseconds.\n", + "2023-09-04 11:11:14.036528: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:237] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", + "2023-09-04 11:11:14.046951: I tensorflow/compiler/mlir/lite/flatbuffer_export.cc:1963] Estimated count of arithmetic ops: 0.101 M ops, equivalently 0.050 M MACs\n", + "\n", + "Estimated count of arithmetic ops: 0.101 M ops, equivalently 0.050 M MACs\n", + "\u001b[0m" + ] + } + ], + "source": [ + "# Step 3: Train model\n", + "\n", + "!{sys.executable} ../openwakeword/train.py --training_config my_model.yaml --train_model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5b0eed06", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "b069686e", + "metadata": { + "ExecuteTime": { + "end_time": "2023-09-04T13:56:23.163906Z", + "start_time": "2023-09-04T13:56:23.027821Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "negative_features_test.npy negative_train\t\t positive_test\r\n", + "negative_features_train.npy positive_features_test.npy positive_train\r\n", + "negative_test\t\t positive_features_train.npy\r\n" + ] + } + ], + "source": [ + "!ls generated_data/my_model/" ] } ], diff --git a/openwakeword/data.py b/openwakeword/data.py index 184dfb5..0ea5344 100755 --- a/openwakeword/data.py +++ b/openwakeword/data.py @@ -826,7 +826,6 @@ class mmap_batch_generator: # Restart at zeroth index if an array reaches the end if self.data_counter[label] >= self.shapes[label][0]: self.data_counter[label] = 0 - # self.data[label] = np.load(self.data_files[label], mmap_mode='r') # Get data from mmaped file x = self.data[label][self.data_counter[label]:self.data_counter[label]+n] diff --git a/openwakeword/train.py b/openwakeword/train.py index a8c1c75..9019df7 100755 --- a/openwakeword/train.py +++ b/openwakeword/train.py @@ -201,7 +201,7 @@ class Model(nn.Module): val_set_hrs = 11.3 # Sequence 1 - print("Starting training sequence 1...") + logging.info("Starting training sequence 1...") lr = 0.0001 weights = np.linspace(1, max_negative_weight, int(steps)).tolist() val_steps = np.linspace(steps-int(steps*0.25), steps, 20).astype(np.int64) @@ -216,14 +216,14 @@ class Model(nn.Module): target_val_accuracy=target_val_accuracy, target_val_recall=target_val_recall) # Sequence 2 - print("Starting training sequence 2...") + logging.info("Starting training sequence 2...") lr = lr/10 steps = steps/10 # Adjust weights as needed based on false positive per hour performance from first sequence if self.best_val_fp > target_val_fp_per_hour: max_negative_weight = max_negative_weight*2 - print("Increasing weight on negative examples to reduce false positives...") + logging.info("Increasing weight on negative examples to reduce false positives...") weights = np.linspace(1, max_negative_weight, int(steps)).tolist() val_steps = np.linspace(1, steps, 20).astype(np.int16) @@ -238,13 +238,13 @@ class Model(nn.Module): target_val_accuracy=target_val_accuracy, target_val_recall=target_val_recall) # Sequence 3 - print("Starting training sequence 3...") + logging.info("Starting training sequence 3...") lr = lr/10 # Adjust weights as needed based on false positive per hour performance from second sequence if self.best_val_fp > target_val_fp_per_hour: max_negative_weight = max_negative_weight*2 - print("Increasing weight on negative examples to reduce false positives...") + logging.info("Increasing weight on negative examples to reduce false positives...") weights = np.linspace(1, max_negative_weight, int(steps)).tolist() val_steps = np.linspace(1, steps, 20).astype(np.int16) @@ -259,10 +259,16 @@ class Model(nn.Module): target_val_accuracy=target_val_accuracy, target_val_recall=target_val_recall) # Merge best models - print("Merging best checkpoints into single model...") - combined_model = self.average_models(models=self.best_models) + if len(self.best_models) == 0: + logging.warning("No checkpoint with metrics >= than target values was found!\n" + "Consider generating more examples, or reducing target metrics." + "Returning the model corresponding to the last training step.\n\n") + return self.model + else: + logging.info("Merging best checkpoints into single model...") + combined_model = self.average_models(models=self.best_models) - # Report validationmetrics for combined model + # Report validation metrics for combined model with torch.no_grad(): for batch in X_val: x, y = batch[0].to(self.device), batch[1].to(self.device) @@ -279,11 +285,9 @@ class Model(nn.Module): combined_model_fp_per_hr = (combined_model_fp/val_set_hrs).detach().cpu().numpy() - print("\n################\n") - print("Final Model Accuracy:", combined_model_accuracy) - print("Final Model Recall:", combined_model_recall) - print("Final Model False Positives per Hour:", combined_model_fp_per_hr) - print("\n################\n") + logging.info(f"\n################\nFinal Model Accuracy: {combined_model_accuracy}" + f"\nFinal Model Recall: {combined_model_recall}\nFinal Model False Positives per Hour: {combined_model_fp_per_hr}" + "\n################\n") return combined_model @@ -295,6 +299,7 @@ class Model(nn.Module): "Use the `export_to_onnx` function instead.") # Save ONNX model + logging.info(f"Saving ONNX mode as '{os.path.join(output_dir, model_name + '.onnx')}'") model_to_save = copy.deepcopy(model) torch.onnx.export(model_to_save.to("cpu"), torch.rand(self.input_shape)[None, ], os.path.join(output_dir, model_name + ".onnx")) @@ -318,6 +323,7 @@ class Model(nn.Module): converter = tf.lite.TFLiteConverter.from_saved_model(os.path.join(tmp_dir, "tf_model")) tflite_model = converter.convert() + logging.info(f"Saving tflite mode as '{os.path.join(output_dir, model_name + '.tflite')}'") with open(os.path.join(output_dir, model_name + ".tflite"), 'wb') as f: f.write(tflite_model) @@ -415,11 +421,11 @@ class Model(nn.Module): self.history["val_recall"].append(val_recall) # Save models with a validation score below a given threshold - print(val_fp_per_hr, self.history["val_accuracy"][-1], self.history["val_recall"][-1]) + # print(val_fp_per_hr, self.history["val_accuracy"][-1], self.history["val_recall"][-1]) if val_fp_per_hr <= max(self.best_val_fp, max_val_fp_per_hr) and \ self.history["val_accuracy"][-1] >= target_val_accuracy and \ self.history["val_recall"][-1] >= target_val_recall: - print("Saving checkpoint with metrics >= to targets!") + # logging.info("Saving checkpoint with metrics >= to targets!") self.best_models.append(copy.deepcopy(self.model)) self.best_val_fp = val_fp_per_hr self.best_val_recall = self.history["val_recall"][-1] @@ -441,7 +447,6 @@ if __name__ == '__main__': parser.add_argument( "--generate_clips", help="Execute the synthetic data generation process", - type=bool, action="store_true", default="False", required=False @@ -449,7 +454,6 @@ if __name__ == '__main__': parser.add_argument( "--augment_clips", help="Execute the synthetic data augmentation process", - type=bool, action="store_true", default="False", required=False @@ -457,7 +461,6 @@ if __name__ == '__main__': parser.add_argument( "--train_model", help="Execute the model training process", - type=bool, action="store_true", default="False", required=False @@ -471,8 +474,11 @@ if __name__ == '__main__': from generate_samples import generate_samples # Define output locations + config["output_dir"] = os.path.abspath(config["output_dir"]) if not os.path.exists(config["output_dir"]): os.mkdir(config["output_dir"]) + if not os.path.exists(os.path.join(config["output_dir"], config["model_name"])): + os.mkdir(os.path.join(config["output_dir"], config["model_name"])) positive_train_output_dir = os.path.join(config["output_dir"], config["model_name"], "positive_train") positive_test_output_dir = os.path.join(config["output_dir"], config["model_name"], "positive_test") @@ -486,10 +492,13 @@ if __name__ == '__main__': if args.generate_clips is True: # Generate positive clips for training + logging.info("Generating positive clips for training") + if not os.path.exists(positive_train_output_dir): + os.mkdir(positive_train_output_dir) n_current_samples = len(os.listdir(positive_train_output_dir)) if n_current_samples <= 0.95*config["n_samples"]: generate_samples( - text=[config["target_phrase"]], max_samples=config["n_samples"]-n_current_samples, + text=config["target_phrase"], max_samples=config["n_samples"]-n_current_samples, batch_size=config["tts_batch_size"], noise_scales=[0.98], noise_scale_ws=[0.98], length_scales=[0.75, 1.0, 1.25], output_dir=positive_train_output_dir, auto_reduce_batch_size=True, @@ -500,50 +509,59 @@ if __name__ == '__main__': logging.warning(f"Skipping generation of positive clips for training, as ~{config['n_samples']} already exist") # Generate positive clips for testing + logging.info("Generating positive clips for testing") + if not os.path.exists(positive_test_output_dir): + os.mkdir(positive_test_output_dir) n_current_samples = len(os.listdir(positive_test_output_dir)) if n_current_samples <= 0.95*config["n_samples_val"]: - generate_samples(text=[config["target_phrase"]], max_samples=config["n_samples_val"]-n_current_samples, - batch_size=config["tts_batch_size"], - noise_scales=[1.0], noise_scale_ws=[1.0], length_scales=[0.75, 1.0, 1.25], - output_dir=positive_test_output_dir, auto_reduce_batch_size=True) + generate_samples(text=config["target_phrase"], max_samples=config["n_samples_val"]-n_current_samples, + batch_size=config["tts_batch_size"], + noise_scales=[1.0], noise_scale_ws=[1.0], length_scales=[0.75, 1.0, 1.25], + output_dir=positive_test_output_dir, auto_reduce_batch_size=True) torch.cuda.empty_cache() else: logging.warning(f"Skipping generation of positive clips testing, as ~{config['n_samples_val']} already exist") # Generate adversarial negative clips for training + logging.info("Generating negative clips for training") + if not os.path.exists(negative_train_output_dir): + os.mkdir(negative_train_output_dir) n_current_samples = len(os.listdir(negative_train_output_dir)) if n_current_samples <= 0.95*config["n_samples"]: adversarial_texts = config["custom_negative_phrases"] for target_phrase in config["target_phrase"]: - adversarial_texts.append(generate_adversarial_texts( + adversarial_texts.extend(generate_adversarial_texts( input_text=target_phrase, N=config["n_samples"]//len(config["target_phrase"]), include_partial_phrase=1.0, include_input_words=0.2)) generate_samples(text=adversarial_texts, max_samples=config["n_samples"]-n_current_samples, - batch_size=config["tts_batch_size"]//7, - noise_scales=[0.98], noise_scale_ws=[0.98], length_scales=[0.75, 1.0, 1.25], - output_dir=negative_train_output_dir, auto_reduce_batch_size=True, - file_names=[uuid.uuid4().hex + ".wav" for i in range(config["n_samples"])] - ) + batch_size=config["tts_batch_size"]//7, + noise_scales=[0.98], noise_scale_ws=[0.98], length_scales=[0.75, 1.0, 1.25], + output_dir=negative_train_output_dir, auto_reduce_batch_size=True, + file_names=[uuid.uuid4().hex + ".wav" for i in range(config["n_samples"])] + ) torch.cuda.empty_cache() else: logging.warning(f"Skipping generation of negative clips for training, as ~{config['n_samples']} already exist") # Generate adversarial negative clips for testing + logging.info("Generating negative clips for testing") + if not os.path.exists(negative_test_output_dir): + os.mkdir(negative_test_output_dir) n_current_samples = len(os.listdir(negative_test_output_dir)) if n_current_samples <= 0.95*config["n_samples_val"]: adversarial_texts = config["custom_negative_phrases"] for target_phrase in config["target_phrase"]: - adversarial_texts.append(generate_adversarial_texts( + adversarial_texts.extend(generate_adversarial_texts( input_text=target_phrase, N=config["n_samples_val"]//len(config["target_phrase"]), include_partial_phrase=1.0, include_input_words=0.2)) generate_samples(text=adversarial_texts, max_samples=config["n_samples_val"]-n_current_samples, - batch_size=config["tts_batch_size"]//7, - noise_scales=[1.0], noise_scale_ws=[1.0], length_scales=[0.75, 1.0, 1.25], - output_dir=negative_test_output_dir, auto_reduce_batch_size=True) + batch_size=config["tts_batch_size"]//7, + noise_scales=[1.0], noise_scale_ws=[1.0], length_scales=[0.75, 1.0, 1.25], + output_dir=negative_test_output_dir, auto_reduce_batch_size=True) torch.cuda.empty_cache() else: logging.warning(f"Skipping generation of negative clips for testing, as ~{config['n_samples_val']} already exist") @@ -551,34 +569,34 @@ if __name__ == '__main__': # Do Data Augmentation if args.augment_clips is True: if not os.path.exists(os.path.join(feature_save_dir, "positive_features_train.npy")): - logging.info("Augmenting generated clips...") + logging.info("Creating augmentation generators") positive_clips_train = [str(i) for i in Path(positive_train_output_dir).glob("*.wav")]*config["augmentation_rounds"] positive_clips_train_generator = augment_clips(positive_clips_train, total_length=config["total_length"], - batch_size=config["augmentation_batch_size"], - background_clip_paths=background_paths, - RIR_paths=rir_paths) + batch_size=config["augmentation_batch_size"], + background_clip_paths=background_paths, + RIR_paths=rir_paths) positive_clips_test = [str(i) for i in Path(positive_test_output_dir).glob("*.wav")]*config["augmentation_rounds"] positive_clips_test_generator = augment_clips(positive_clips_test, total_length=config["total_length"], - batch_size=config["augmentation_batch_size"], - background_clip_paths=background_paths, - RIR_paths=rir_paths) + batch_size=config["augmentation_batch_size"], + background_clip_paths=background_paths, + RIR_paths=rir_paths) negative_clips_train = [str(i) for i in Path(negative_train_output_dir).glob("*.wav")]*config["augmentation_rounds"] negative_clips_train_generator = augment_clips(negative_clips_train, total_length=config["total_length"], - batch_size=config["augmentation_batch_size"], - background_clip_paths=background_paths, - RIR_paths=rir_paths) + batch_size=config["augmentation_batch_size"], + background_clip_paths=background_paths, + RIR_paths=rir_paths) negative_clips_test = [str(i) for i in Path(negative_test_output_dir).glob("*.wav")]*config["augmentation_rounds"] negative_clips_test_generator = augment_clips(negative_clips_test, total_length=config["total_length"], - batch_size=config["augmentation_batch_size"], - background_clip_paths=background_paths, - RIR_paths=rir_paths) + batch_size=config["augmentation_batch_size"], + background_clip_paths=background_paths, + RIR_paths=rir_paths) # Compute features and save to disk via memmapped arrays - logging.info("Computer openwakeword features for generated samples...") + logging.info("Computing openwakeword features for generated samples") n_cpus = os.cpu_count() if n_cpus is None: n_cpus = 1 @@ -588,25 +606,25 @@ if __name__ == '__main__': clip_duration=config["total_length"], output_file=os.path.join(feature_save_dir, "positive_features_train.npy"), device="gpu" if torch.cuda.is_available() else "cpu", - ncpu=n_cpus if not torch.cuda_is_available() else 1) + ncpu=n_cpus if not torch.cuda.is_available() else 1) compute_features_from_generator(negative_clips_train_generator, n_total=len(os.listdir(negative_train_output_dir)), clip_duration=config["total_length"], output_file=os.path.join(feature_save_dir, "negative_features_train.npy"), device="gpu" if torch.cuda.is_available() else "cpu", - ncpu=n_cpus if not torch.cuda_is_available() else 1) + ncpu=n_cpus if not torch.cuda.is_available() else 1) compute_features_from_generator(positive_clips_test_generator, n_total=len(os.listdir(positive_test_output_dir)), clip_duration=config["total_length"], output_file=os.path.join(feature_save_dir, "positive_features_test.npy"), device="gpu" if torch.cuda.is_available() else "cpu", - ncpu=n_cpus if not torch.cuda_is_available() else 1) + ncpu=n_cpus if not torch.cuda.is_available() else 1) compute_features_from_generator(negative_clips_test_generator, n_total=len(os.listdir(negative_test_output_dir)), clip_duration=config["total_length"], output_file=os.path.join(feature_save_dir, "negative_features_test.npy"), device="gpu" if torch.cuda.is_available() else "cpu", - ncpu=n_cpus if not torch.cuda_is_available() else 1) + ncpu=n_cpus if not torch.cuda.is_available() else 1) else: logging.warning("Openwakeword features already exist, skipping data augmentation and feature generation") @@ -618,18 +636,28 @@ if __name__ == '__main__': oww = Model(n_classes=1, input_shape=input_shape, model_type=config["model_type"], layer_dim=config["layer_size"], seconds_per_example=1280*input_shape[0]/16000) - # Create data and label transform functions for batch generation + # Create data transform function for batch generation to handle differ clip lengths (todo: write tests for this) def f(x, n=16): """Simple transformation function to ensure negative data is the appropriate shape for the model size""" - n_chunks = x.shape[1]//n - stacked = np.vstack(( - [x[:, i:i+n, :] for i in range(n_chunks)] - )) - return stacked + if n > x.shape[1] or n < x.shape[1]: + x = np.vstack(x) + new_batch = np.array([x[i:i+n, :] for i in range(0, x.shape[0]-n, n)]) + else: + return x + return new_batch - data_transforms = {key: f for key in config["negative_data_files"].keys()} - label_transforms = {key: lambda x: [1 for i in x] if key == "positive" else lambda x: [0 for i in x] - for key in ["positive"] + config["negative_data_files"] + ["adversarial_negative"]} + # Create label transforms as needed for model (currently only supports binary classification models) + data_transforms = {key: f for key in config["feature_data_files"].keys()} + label_transforms = {} + for key in ["positive"] + list(config["feature_data_files"].keys()) + ["adversarial_negative"]: + if key == "positive": + label_transforms[key] = lambda x: [1 for i in x] + else: + label_transforms[key] = lambda x: [0 for i in x] + + # Add generated positive and adversarial negative clips to the feature data files dictionary + config["feature_data_files"]['positive'] = os.path.join(feature_save_dir, "positive_features_train.npy") + config["feature_data_files"]['adversarial_negative'] = os.path.join(feature_save_dir, "negative_features_train.npy") # Make PyTorch data loaders for training and validation data batch_generator = mmap_batch_generator( @@ -646,8 +674,13 @@ if __name__ == '__main__': def __iter__(self): return self.generator + n_cpus = os.cpu_count() + if n_cpus is None: + n_cpus = 1 + else: + n_cpus = n_cpus//2 X_train = torch.utils.data.DataLoader(IterDataset(batch_generator), - batch_size=None, num_workers=8, prefetch_factor=16) + batch_size=None, num_workers=n_cpus, prefetch_factor=16) X_val_fp = np.load(config["false_positive_validation_data_path"]) X_val_fp = np.array([X_val_fp[i:i+input_shape[0]] for i in range(0, X_val_fp.shape[0]-input_shape[0], 1)]) # reshape to match model @@ -668,13 +701,6 @@ if __name__ == '__main__': batch_size=len(labels) ) - # Run auto training and save model - steps = 100000 - max_neg_weight = 1500 - target_accuracy = 0.7 - target_recall = 0.5 - target_fp_per_hour = 0.2 - # Run auto training best_model = oww.auto_train( X_train=X_train, @@ -684,7 +710,7 @@ if __name__ == '__main__': max_negative_weight=config["max_negative_weight"], target_val_accuracy=config["target_accuracy"], target_val_recall=config["target_recall"], - target_val_fp_per_hour=config["target_fp_per_hour"] + target_val_fp_per_hour=config["target_false_positives_per_hour"] ) # Export the trained model to onnx and tflite formats