From 635d029d29c01e00ed88bd26155a9ff73fd198de Mon Sep 17 00:00:00 2001
From: dscripka <david.scripka@gmail.com>
Date: Sun, 30 Apr 2023 18:50:09 -0400
Subject: [PATCH] First version of minimal WebUI to manage cached files [skip
 ci]

---
 examples/detect_from_microphone.py      |  76 +++++++---
 openwakeword/custom_verifier_model.py   |   2 +-
 openwakeword/model.py                   |  60 +++++---
 openwakeword/resources/webui/index.html | 178 ++++++++++++++++++++++++
 openwakeword/resources/webui/server.py  |  53 +++++++
 openwakeword/resources/webui/style.css  |  68 +++++++++
 openwakeword/utils.py                   |   4 +-
 setup.py                                |   2 +-
 8 files changed, 399 insertions(+), 44 deletions(-)
 create mode 100644 openwakeword/resources/webui/index.html
 create mode 100644 openwakeword/resources/webui/server.py
 create mode 100644 openwakeword/resources/webui/style.css

diff --git a/examples/detect_from_microphone.py b/examples/detect_from_microphone.py
index f72b084..a62778b 100644
--- a/examples/detect_from_microphone.py
+++ b/examples/detect_from_microphone.py
@@ -13,10 +13,15 @@
 # limitations under the License.
 
 # Imports
+import sys
+import os
 import pyaudio
 import numpy as np
 from openwakeword.model import Model
+from openwakeword.resources.webui.server import openWakeWordWebUI
 import argparse
+from http.server import ThreadingHTTPServer
+import threading
 
 # Parse input arguments
 parser=argparse.ArgumentParser()
@@ -25,7 +30,20 @@ parser.add_argument(
     help="How much audio (in samples) to predict on at once",
     type=int,
     default=1280,
-    required=True
+    required=False
+)
+
+parser.add_argument(
+    "--vad_threshold",
+    help="The minimum threshold for voice activity detection required before an activations",
+    type=float,
+    default=0.3
+)
+
+parser.add_argument(
+    "--custom_verifier_model_online_learning",
+    help="Whether to enable online learning of a custom verifier model",
+    action='store_true',
 )
 
 args=parser.parse_args()
@@ -39,8 +57,10 @@ audio = pyaudio.PyAudio()
 mic_stream = audio.open(format=FORMAT, channels=CHANNELS, rate=RATE, input=True, frames_per_buffer=CHUNK)
 
 # Load pre-trained openwakeword models
-owwModel = Model()
-
+owwModel = Model(
+    vad_threshold=args.vad_threshold,
+    custom_verifier_model_online_learning=args.custom_verifier_model_online_learning
+)
 # Run capture loop continuosly, checking for wakewords
 if __name__ == "__main__":
     # Generate output string header
@@ -50,28 +70,40 @@ if __name__ == "__main__":
     print("#"*100)
     print("\n"*13)
 
+    # Start HTTP server
+    os.chdir(owwModel.cache_dir)
+    server = ThreadingHTTPServer(("127.0.0.1", 9999), lambda *args, **kwargs: openWakeWordWebUI(owwModel, *args, **kwargs))
+    server_thread = threading.Thread(target=server.serve_forever)
+    server_thread.daemon = True
+    server_thread.start()
+
     while True:
-        # Get audio
-        audio = np.frombuffer(mic_stream.read(CHUNK), dtype=np.int16)
+        try:
+            # Get audio
+            audio = np.frombuffer(mic_stream.read(CHUNK), dtype=np.int16)
 
-        # Feed to openWakeWord model
-        prediction = owwModel.predict(audio)
+            # Feed to openWakeWord model
+            prediction = owwModel.predict(audio)
 
-        # Column titles
-        n_spaces = 16
-        output_string_header = """
-            Model Name         | Score | Wakeword Status
-            --------------------------------------
-            """
+            # Column titles
+            n_spaces = 16
+            output_string_header = """
+                Model Name         | Score | Wakeword Status
+                --------------------------------------
+                """
 
-        for mdl in owwModel.prediction_buffer.keys():
-            # Add scores in formatted table
-            scores = list(owwModel.prediction_buffer[mdl])
-            curr_score = format(scores[-1], '.20f').replace("-", "")
+            for mdl in owwModel.prediction_buffer.keys():
+                # Add scores in formatted table
+                scores = list(owwModel.prediction_buffer[mdl])
+                curr_score = format(scores[-1], '.20f').replace("-", "")
 
-            output_string_header += f"""{mdl}{" "*(n_spaces - len(mdl))}   | {curr_score[0:5]} | {"--"+" "*20 if scores[-1] <= 0.5 else "Wakeword Detected!"}
-            """
+                output_string_header += f"""{mdl}{" "*(n_spaces - len(mdl))}   | {curr_score[0:5]} | {"--"+" "*20 if scores[-1] <= 0.5 else "Wakeword Detected!"}
+                """
 
-        # Print results table
-        print("\033[F"*14)
-        print(output_string_header, "                             ", end='\r')
+            # # Print results table
+            # print("\033[F"*15)
+            # print(output_string_header, "                             ", end='\r')
+        
+        except KeyboardInterrupt:
+            server.shutdown()
+            sys.exit(0)
diff --git a/openwakeword/custom_verifier_model.py b/openwakeword/custom_verifier_model.py
index 4b3b922..dc5eb91 100644
--- a/openwakeword/custom_verifier_model.py
+++ b/openwakeword/custom_verifier_model.py
@@ -92,7 +92,7 @@ def flatten_features(x):
 
 def make_sklearn_pipeline():
     # clf = SVC(gamma='auto', probability=True)
-    clf = LogisticRegression(random_state=0, max_iter=2000, C=0.01)
+    clf = LogisticRegression(random_state=0, max_iter=2000, C=0.01, class_weight='balanced')
     pipeline = make_pipeline(FunctionTransformer(flatten_features), StandardScaler(), clf)
     return pipeline
 
diff --git a/openwakeword/model.py b/openwakeword/model.py
index 57640b4..d3f632a 100755
--- a/openwakeword/model.py
+++ b/openwakeword/model.py
@@ -19,6 +19,8 @@ import openwakeword
 from openwakeword.utils import AudioFeatures
 
 import wave
+import soundfile
+import uuid
 import os
 from pathlib import Path
 import pickle
@@ -125,12 +127,17 @@ class Model():
             self.model_input_names[mdl_name] = self.models[mdl_name].get_inputs()[0].name
 
             # Create attributes to store realtime usage data for verifier models
-            self.custom_verifier_data[mdl_name] = {
-                "features": {
-                    "positive": deque(maxlen=1000),
-                    "negative": deque(maxlen=1000)
+            usage_data = {
+                    "features": {
+                        "positive": deque(maxlen=1000),
+                        "negative": deque(maxlen=2000)
+                    }
                 }
-            }
+            if self.model_outputs[mdl_name] == 1:
+                self.custom_verifier_data[mdl_name] = usage_data
+            else:
+                for cls in self.class_mapping[mdl_name].values():
+                    self.custom_verifier_data[cls] = usage_data
 
             # Create filesystem cache locations, or load existing data in the cache
             if cache_directory == "":
@@ -140,6 +147,7 @@ class Model():
                 self.cache_dir = cache_directory
             if not os.path.exists(self.cache_dir):
                 os.mkdir(self.cache_dir)
+                os.mkdir(os.path.join(self.cache_dir, "activation_clips"))
             else:
                 if os.path.exists(os.path.join(self.cache_dir, "cached_features.pkl")):
                     # Load existing data in the cache
@@ -284,7 +292,6 @@ class Model():
                         self.custom_verifier_data[cls]["features"]["negative"].append(frame_features.flatten())
 
                     # Update scores of positive predictions
-                    positive_examples_added = False
                     if predictions[cls] >= self.custom_verifier_threshold:
                         parent_model = self.get_parent_model_from_label(cls)
                         if self.custom_verifier_models.get(parent_model, False):
@@ -295,35 +302,52 @@ class Model():
                                 )[0][-1]
                                 predictions[cls] = verifier_prediction
 
-                            # Update data cache for verifier models
-                            if self.custom_verifier_model_online_learning and predictions[cls] >= self.custom_verifier_threshold:
+                            # Update data cache for verifier models only on predictions with two or more sequential
+                            # frames with scores above the threshold
+                            if self.custom_verifier_model_online_learning and predictions[cls] >= self.custom_verifier_threshold and \
+                               self.prediction_buffer[cls][-1] >= self.custom_verifier_threshold:
                                 self.custom_verifier_data[cls]["features"]["positive"].append(frame_features.flatten())
-                                positive_examples_added = True
 
-                                # Save feature data to cache
+                            # Save feature data to cache after activation has finished
+                            if (np.array(self.prediction_buffer[cls])[-3:] > self.custom_verifier_threshold).sum() == 0:
                                 pickle.dump(self.custom_verifier_data, open(os.path.join(self.cache_dir, "cached_features.pkl"), 'wb'))
 
-                    # Train verifier model on latest data at most every 10 seconds or after every positive detection
+                                # Save audio clip of activation (last 4 seconds)
+                                raw_data = np.array(list(self.preprocessor.raw_data_buffer)[-16000*4:]).astype(np.int16)
+                                soundfile.write(os.path.join(self.cache_dir, "activation_clips", f"{cls}_{uuid.uuid4().hex[0:6]}.ogg"),
+                                                raw_data, 16000)
+
+                    # Train verifier model on latest data at most every 60 seconds and only if there are enough positive clips
                     if self.custom_verifier_model_online_learning and \
-                       (self.samples_processed >= 16000*10 or positive_examples_added):
+                       self.samples_processed >= 16000*60 and \
+                       len(self.custom_verifier_data[cls]["features"]["positive"]) >= 3:
                         self.samples_processed = 0
                         parent_model = self.get_parent_model_from_label(cls)
+                        # y = np.array(
+                        #     [1]*np.array(self.custom_verifier_data[cls]["features"]["positive"]).shape[0] +
+                        #     [0]*np.array(self.custom_verifier_data[cls]["features"]["negative"]).shape[0]
+                        # )
+
                         y = np.array(
                             [1]*np.array(self.custom_verifier_data[cls]["features"]["positive"]).shape[0] +
-                            [0]*np.array(self.custom_verifier_data[cls]["features"]["negative"]).shape[0]
+                            [0]*np.array(self.custom_verifier_data[cls]["features"]["positive"]).shape[0]*2
                         )
 
-                        # need a minimum number of examples to train (5)
-                        if len(self.custom_verifier_data[cls]["features"]["positive"]) > 3 and \
-                           len(self.custom_verifier_data[cls]["features"]["negative"]) > 5:
+                        # need a minimum number of negative examples to train
+                        if len(self.custom_verifier_data[cls]["features"]["negative"]) > len(self.custom_verifier_data[cls]["features"]["positive"]):
                             print("Updating custom verifier model")
+                            random_ndcs = np.random.randint(
+                                0, len(self.custom_verifier_data[cls]["features"]["negative"]) - len(self.custom_verifier_data[cls]["features"]["positive"]) - 1,
+                                len(self.custom_verifier_data[cls]["features"]["positive"]))
+
                             x = np.vstack((
                                 np.array(self.custom_verifier_data[cls]["features"]["positive"]),
-                                np.array(self.custom_verifier_data[cls]["features"]["negative"]),
+                                np.array(self.custom_verifier_data[cls]["features"]["negative"])[random_ndcs, :],
+                                np.array(self.custom_verifier_data[cls]["features"]["negative"])[-len(random_ndcs):, :]
                             ))
                             self.custom_verifier_models[parent_model].fit(x, y)
 
-                            print("Accuracy: ", sum(y == self.custom_verifier_models[parent_model].predict(x))/len(x))
+                            # print("Accuracy: ", sum(y == self.custom_verifier_models[parent_model].predict(x))/len(x))
 
             # Update prediction buffer, and zero predictions for first 5 frames during model initialization
             for cls in predictions.keys():
diff --git a/openwakeword/resources/webui/index.html b/openwakeword/resources/webui/index.html
new file mode 100644
index 0000000..279607a
--- /dev/null
+++ b/openwakeword/resources/webui/index.html
@@ -0,0 +1,178 @@
+<!-- https://www.w3schools.com/howto/howto_js_tabs.asp -->
+<!-- https://www.w3schools.com/howto/howto_js_filter_table.asp -->
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>openWakeWord WebUI</title>
+    <link rel="stylesheet" type="text/css" href="/style.css"/>
+  </head>
+  <body>
+
+    <!-- Set tabs for different content -->
+    <div class="tab">
+      <button class="tablinks" onclick="openTab(event, 'review_clips')" id="review_clips_button">Cached Audio Clips</button>
+      <button class="tablinks" onclick="openTab(event, 'streaming_predictions')">Realtime Predictions</button>
+    </div>
+
+    <!-- Create tab for reviewing cached clips -->
+    <div id="review_clips" class="tabcontent">
+      <h3>These are the cached clips of activations from the openWakeWord models, which are used when training the custom verifier models. Review and delete false positives as needed to improve performance of the verifier models.</h3>
+      <input type="text" id="filter_input" onkeyup="filterRows()" placeholder="Filter by name...">
+      <table id="audio-table">
+        <thead>
+          <tr>
+            <th>Name</th>
+            <th>Audio</th>
+            <th></th>
+          </tr>
+        </thead>
+        <tbody></tbody>
+      </table>
+    </div>
+
+    <!-- Tab for realtime prediction results -->
+    <div id="streaming_predictions" class="tabcontent">
+      <div>
+        <button onclick="plotModelPredictions('model1')">Model 1</button>
+        <button onclick="plotModelPredictions('model1')">Model 1</button>
+        <button onclick="plotModelPredictions('model1')">Model 1</button>
+      </div>
+      <canvas id="myCanvas" width="800" height="400"></canvas>
+    </div>
+
+    <script>
+      // Load default tab
+      document.getElementById("review_clips_button").click();
+
+      // Make a GET request to receive the names and audio data for all of the files
+      fetch('/list_cache_files')
+        .then(response => response.json())
+        .then(audioFiles => {
+          const tbody = document.querySelector('#audio-table tbody');
+
+          // Loop through each audio file and add a row to the table
+          audioFiles.forEach((audioFile, index) => {
+            const row = document.createElement('tr');
+
+            // Add filename and audio player to the table
+            const filenameCell = document.createElement('td');
+            filenameCell.textContent = audioFile.filename;
+            filenameCell.id = audioFile.filename;
+            row.appendChild(filenameCell);
+
+            const audioPlayer = document.createElement('audio');
+            audioPlayer.controls = true;
+            audioPlayer.src = audioFile.data;
+            row.appendChild(audioPlayer);
+
+            // Add delete button for each clip
+            const deleteButton = document.createElement('td')
+            // deleteCell.innerHTML = "<a href='/delete_clip?filename=" + audioFile.filename + "'>&#x2716;</a>"
+            deleteButton.innerHTML = '<u style="text-decoration-color:red" style><span style="color:red;">Delete</span><u>'
+            deleteButton.addEventListener('click', function (){
+              alert('Are you sure? This action can not be reversed.');
+
+              // Send GET request to delete file
+              fetch('/delete_clip?filename=' + audioFile.filename)
+
+              // Delete row from table
+              var table = document.getElementById("audio-table");
+              table.deleteRow(this.parentNode.rowIndex)
+            })
+            row.appendChild(deleteButton)
+
+            tbody.appendChild(row);
+          });
+        });
+
+      // Define the data points for the line plot
+      const data = [0.0, .2, .3, .4, .8, 1.0];
+
+      // Draw canvas data
+      const canvas = document.getElementById("myCanvas");
+      const ctx = canvas.getContext("2d");
+      ctx.canvas.width = window.innerWidth*0.9;
+
+      // Set up the x and y scales
+      const label_offset = 45;
+      const h = canvas.height*0.9;
+      const w = canvas.width - label_offset;
+      const xScale = w / (data.length - 1);
+      const yScale = h*0.9;
+
+      // Draw y-axis with labels
+      ctx.fillStyle = "black"; // set the color of the labels to black
+      ctx.textAlign = "right"; // align labels to right of y-axis
+      ctx.textBaseline = "middle"; // center labels vertically
+      const yLabels = [0, 0.25, 0.5, 0.75, 1.0]; // set values for y-axis labels
+      yLabels.forEach(label => {
+        const xPos = 30; // x position of label (adjust as needed)
+        const yPos = h - label * yScale; // y position of label
+        ctx.fillText(label.toString(), xPos, yPos);
+        ctx.beginPath();
+        ctx.moveTo(40, yPos);
+        ctx.lineTo(canvas.width, yPos);
+        ctx.strokeStyle = "#ccc";
+        ctx.stroke();
+      })
+      
+      // Define function to draw series
+      function plot_data(canvas, data, color) {        
+        // Draw the line plot
+        ctx.beginPath();
+        ctx.moveTo(0 + label_offset, h - data[0] * yScale);
+        for (let i = 1; i < data.length; i++) {
+          ctx.lineTo(i * xScale + label_offset, h - data[i] * yScale);
+        }
+        ctx.strokeStyle = color; 
+        ctx.stroke();
+      }
+      
+      plot_data(canvas, data, "red")
+
+      function filterRows() {
+        // Declare variables
+        var input, filter, table, tr, td, i, txtValue;
+        input = document.getElementById("filter_input");
+        filter = input.value.toUpperCase();
+        table = document.getElementById("audio-table");
+        tr = table.getElementsByTagName("tr");
+
+        // Loop through all table rows, and hide those who don't match the search query
+        for (i = 0; i < tr.length; i++) {
+          td = tr[i].getElementsByTagName("td")[0];
+          if (td) {
+            txtValue = td.textContent || td.innerText;
+            if (txtValue.toUpperCase().indexOf(filter) > -1) {
+              tr[i].style.display = "";
+            } else {
+              tr[i].style.display = "none";
+            }
+          }
+        }
+      }
+
+      function openTab(evt, tabName) {
+        // Declare all variables
+        var i, tabcontent, tablinks;
+
+        // Get all elements with class="tabcontent" and hide them
+        tabcontent = document.getElementsByClassName("tabcontent");
+        for (i = 0; i < tabcontent.length; i++) {
+          tabcontent[i].style.display = "none";
+        }
+
+        // Get all elements with class="tablinks" and remove the class "active"
+        tablinks = document.getElementsByClassName("tablinks");
+        for (i = 0; i < tablinks.length; i++) {
+          tablinks[i].className = tablinks[i].className.replace(" active", "");
+        }
+
+        // Show the current tab, and add an "active" class to the button that opened the tab
+        document.getElementById(tabName).style.display = "block";
+        evt.currentTarget.className += " active";
+      } 
+
+    </script>
+  </body>
+</html>
\ No newline at end of file
diff --git a/openwakeword/resources/webui/server.py b/openwakeword/resources/webui/server.py
new file mode 100644
index 0000000..3af80e5
--- /dev/null
+++ b/openwakeword/resources/webui/server.py
@@ -0,0 +1,53 @@
+from http.server import BaseHTTPRequestHandler, HTTPServer
+import os
+import json
+from pathlib import Path
+import base64
+from urllib.parse import urlparse
+
+WEBUI_CONTENT = open(os.path.join(os.path.dirname(__file__), "index.html"), "r").read()
+WEBUI_CSS = open(os.path.join(os.path.dirname(__file__), "style.css"), "r").read()
+
+class openWakeWordWebUI(BaseHTTPRequestHandler):
+    def __init__(self, custom_object, *args, **kwargs):
+        self.oww_instance = custom_object
+        super().__init__(*args, **kwargs)
+
+    def _set_headers(self):
+        self.send_response(200)
+        self.send_header("Content-type", "text/html")
+        self.end_headers()
+
+    def do_GET(self):
+        if self.path == "/style.css":
+            self.wfile.write(bytes(WEBUI_CSS, "utf8"))
+
+        if self.path == "/":
+            self._set_headers()
+
+            # Load WebUI
+            self.wfile.write(bytes(WEBUI_CONTENT, "utf8"))
+
+        if "/delete_clip" in self.path:
+            self._set_headers()
+            query = urlparse(self.path).query
+            query_params = dict(qc.split("=") for qc in query.split("&"))
+            os.remove(os.path.join(os.getcwd(), "activation_clips", query_params["filename"]))
+
+        if self.path == "/list_cache_files":
+            self._set_headers()
+
+            # Load files and prepare data
+            data = []
+            for i in Path(os.path.join(os.getcwd(), "activation_clips")).glob("**/*.ogg"):
+                audio_bytes = open(i, "rb").read()
+                base64_data = base64.b64encode(audio_bytes).decode('utf-8')
+                data.append(
+                    {
+                        "filename": str(i).split(os.path.sep)[-1],
+                        "duration": 0,
+                        "data": "data:audio/ogg;base64," + base64_data
+                    }
+                )
+            
+            self.wfile.write(bytes(json.dumps(data), "utf8"))
diff --git a/openwakeword/resources/webui/style.css b/openwakeword/resources/webui/style.css
new file mode 100644
index 0000000..3182bc8
--- /dev/null
+++ b/openwakeword/resources/webui/style.css
@@ -0,0 +1,68 @@
+#filter_input {
+    width: 20%; /* Full-width */
+    font-size: 16px; /* Increase font-size */
+    padding: 12px 10px 12px 10px; /* Add some padding */
+    border: 1px solid #ddd; /* Add a grey border */
+    margin-bottom: 10px; /* Add some space below the input */
+}
+  
+#audio-table {
+    border-collapse: collapse; /* Collapse borders */
+    width: 35%; /* Full-width */
+    border: 1px solid #ddd; /* Add a grey border */
+    font-size: 14px; /* Increase font-size */
+}
+
+#audio-table th, #audio-table td {
+    text-align: left; /* Left-align text */
+    padding: 10px; /* Add padding */
+}
+
+#audio-table tr {
+    /* Add a bottom border to all table rows */
+    border-bottom: 1px solid #ddd;
+}
+
+#audio-table tr.header, #audio-table tr:hover {
+    /* Add a grey background color to the table header and on hover */
+    background-color: #f1f1f1;
+}
+
+/* Style the tab */
+.tab {
+    overflow: hidden;
+    border: 1px solid #ccc;
+    background-color: #f1f1f1;
+}
+
+/* Style the buttons that are used to open the tab content */
+.tab button {
+    background-color: inherit;
+    float: left;
+    border: none;
+    outline: none;
+    cursor: pointer;
+    padding: 14px 16px;
+    transition: 0.3s;
+}
+
+/* Change background color of buttons on hover */
+.tab button:hover {
+    background-color: #ddd;
+}
+
+/* Create an active/current tablink class */
+.tab button.active {
+    background-color: #ccc;
+}
+
+/* Style the tab content */
+.tabcontent {
+    display: none;
+    padding: 6px 12px;
+    border: 1px solid #ccc;
+    border-top: none;
+}
+
+/* Click button over delete text */
+.delete { cursor: pointer; }
\ No newline at end of file
diff --git a/openwakeword/utils.py b/openwakeword/utils.py
index 2fab0ac..8234fd9 100644
--- a/openwakeword/utils.py
+++ b/openwakeword/utils.py
@@ -306,11 +306,11 @@ class AudioFeatures():
         self._buffer_raw_data(x)
         self.accumulated_samples += len(x)
 
-        # Only calculate melspectrogram every ~0.5 seconds to significantly increase efficiency
+        # Calculate melspectrograms
         if self.accumulated_samples >= 1280:
             self._streaming_melspectrogram(self.accumulated_samples)
 
-            # Calculate new audio embeddings/features based on update melspectrograms
+            # Calculate new audio embeddings/features based on updated melspectrograms
             for i in np.arange(self.accumulated_samples//1280-1, -1, -1):
                 ndx = -8*i
                 ndx = ndx if ndx != 0 else len(self.melspectrogram_buffer)
diff --git a/setup.py b/setup.py
index fc835ee..0766d91 100644
--- a/setup.py
+++ b/setup.py
@@ -27,7 +27,7 @@ def build_additional_requires():
 setuptools.setup(
     name="openwakeword",
     version="0.3.1",
-    install_requires=['onnxruntime>=1.10.0,<2', 'tqdm>=4.0,<5.0', 'scipy>=1.3,<2', 'scikit-learn>=1,<2'],
+    install_requires=['onnxruntime>=1.10.0,<2', 'tqdm>=4.0,<5.0', 'scipy>=1.3,<2', 'scikit-learn>=1,<2', "soundfile>=0.11.0,<1"],
     extras_require={
         'test': [
                     'pytest>=7.2.0,<8',