From 58ec0943806b5907b2a3218eb0935b1cdf1c92b3 Mon Sep 17 00:00:00 2001
From: David Scripka <david.scripka@gmail.com>
Date: Thu, 9 Nov 2023 08:36:43 -0500
Subject: [PATCH] Added table for detections and styling to websocket example

---
 examples/web/streaming_client.html | 216 ++++++++++++++++++++---------
 examples/web/streaming_server.py   |   6 +-
 2 files changed, 155 insertions(+), 67 deletions(-)
diff --git a/examples/web/streaming_client.html b/examples/web/streaming_client.html
index 3bf5005..44c98d8 100644
--- a/examples/web/streaming_client.html
+++ b/examples/web/streaming_client.html
@@ -1,31 +1,127 @@
-<html>
 <!DOCTYPE html>
 <html lang="en">
 <head>
   <meta charset="UTF-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <title>Websocket Microphone Streaming</title>
+  <style>
+    body {
+      text-align: center;
+      font-family: 'Roboto', sans-serif;
+    }
+    #startButton {
+      padding: 15px 30px;
+      font-size: 18px;
+      background-color: #03A9F4;
+      border: none;
+      border-radius: 4px;
+      color: white;
+      cursor: pointer;
+      outline: none;
+      transition: background-color 0.3s;
+    }
+    #startButton.listening {
+      background-color: #4CAF50;
+    }
+    table {
+      margin: 20px auto;
+      border-collapse: collapse;
+      width: 60%;
+    }
+    th, td {
+      border: 1px solid #E0E0E0;
+      padding: 10px;
+      text-align: left;
+    }
+    th {
+      background-color: #F5F5F5;
+    }
+
+    @keyframes fadeOut {
+      from {
+        opacity: 1;
+      }
+      to {
+        opacity: 0;
+      }
+    }
+
+    .detected-animation {
+      animation: fadeOut 2s forwards;
+    }
+  </style>
 </head>
 <body>
   <h1>Streaming Audio to openWakeWord Using Websockets</h1>
-  <button id="startButton">Start Recording</button>
+  <button id="startButton">Start Listening</button>
+
+  <table>
+    <tr>
+      <th>Wakeword</th>
+      <th>Detected</th>
+    </tr>
+    <tr>
+      <td></td>
+      <td></td>
+    </tr>
+  </table>
 
   <script>
   // Create websocket connection
-  ws = new WebSocket('ws://localhost:9000/ws');
+  const ws = new WebSocket('ws://localhost:9000/ws');
 
   // When the websocket connection is open
   ws.onopen = function() {
     console.log('WebSocket connection is open');
   };
 
-  // Get responses from websocket
+  // Get responses from websocket and display information
   ws.onmessage = (event) => {
     console.log(event.data);
+    const model_payload = JSON.parse(event.data);
+    if ("loaded_models" in model_payload) {
+      // Add loaded models to the rows of the first column in the table, inserting rows as needed
+      const table = document.querySelector('table');
+      const rows = table.querySelectorAll('tr');
+      for (let i = 1; i < model_payload.loaded_models.length + 1; i++) {
+        if (i < rows.length) {
+          const row = rows[i];
+          const cell = row.querySelectorAll('td')[0];
+          cell.textContent = model_payload.loaded_models[i - 1];
+        } else {
+          // Insert extra rows if needed, both column 1 and 2
+          const row = table.insertRow();
+          const cell1 = row.insertCell();
+          const cell2 = row.insertCell();
+          cell1.textContent = model_payload.loaded_models[i - 1];
+          cell2.textContent = '';
+        }
+      }
+
+    }
+
+    if ("activations" in model_payload) {
+      // Add detected wakeword to the rows of the second column in the table
+      const table = document.querySelector('table');
+      const rows = table.querySelectorAll('tr');
+      for (let i = 1; i < rows.length; i++) {
+        // Check for the model name in the first column and add "Detected!" to the second column if they match
+        if (model_payload.activations.includes(rows[i].querySelectorAll('td')[0].textContent)) {
+          const cell = rows[i].querySelectorAll('td')[1];
+          cell.textContent = "Detected!";
+          cell.classList.add('detected-animation'); // animate fade out
+      
+          // Remove the CSS class after the fade out animation ends to reset the state
+          cell.addEventListener('animationend', () => {
+            cell.textContent = '';
+            cell.classList.remove('detected-animation');
+          }, { once: true });
+        }
+      }
+    }
   };
 
   // Create microphone capture stream
-  // Based on the excellent guide here: https://medium.com/@ragymorkos/gettineg-monochannel-16-bit-signed-integer-pcm-audio-samples-from-the-microphone-in-the-browser-8d4abf81164d
   navigator.getUserMedia = navigator.getUserMedia || 
                          navigator.webkitGetUserMedia || 
                          navigator.mozGetUserMedia || 
@@ -35,78 +131,66 @@
   let audioContext;
   let recorder;
   let volume;
-  let sampleRate
+  let sampleRate;
 
-  if (navigator.getUserMedia)
-  {
-    navigator.getUserMedia({audio: true}, function(stream){
-        audioStream = stream;
+  if (navigator.getUserMedia) {
+    navigator.getUserMedia({audio: true}, function(stream) {
+      audioStream = stream;
 
-        // creates the an instance of audioContext
-        const context = window.AudioContext || window.webkitAudioContext;
-        audioContext = new context();
-        
-        // retrieve the current sample rate of microphone the browser is using and send to Python server
-        sampleRate = audioContext.sampleRate;
-        
-        // creates a gain node
-        volume = audioContext.createGain();
-        
-        // creates an audio node from the microphone incoming stream
-        const audioInput = audioContext.createMediaStreamSource(audioStream);
-        
-        // connect the stream to the gain node
-        audioInput.connect(volume);
-        
-        /* From the spec: This value controls how frequently the audioprocess event is
-        dispatched and how many sample-frames need to be processed each call.
-        Lower values for buffer size will result in a lower (better) latency.
-        Higher values will be necessary to avoid audio breakup and glitches */
-        const bufferSize = 4096;
-        recorder = (audioContext.createScriptProcessor || 
-                    audioContext.createJavaScriptNode).call(audioContext, 
-                                                            bufferSize, 
-                                                            1, 
-                                                            1);
+      // creates the an instance of audioContext
+      const context = window.AudioContext || window.webkitAudioContext;
+      audioContext = new context();
+      
+      // retrieve the current sample rate of microphone the browser is using and send to Python server
+      sampleRate = audioContext.sampleRate;
+      
+      // creates a gain node
+      volume = audioContext.createGain();
+      
+      // creates an audio node from the microphone incoming stream
+      const audioInput = audioContext.createMediaStreamSource(audioStream);
+      
+      // connect the stream to the gain node
+      audioInput.connect(volume);
+      
+      const bufferSize = 4096;
+      recorder = (audioContext.createScriptProcessor || 
+                  audioContext.createJavaScriptNode).call(audioContext, 
+                                                          bufferSize, 
+                                                          1, 
+                                                          1);
 
-                                                            const leftChannel = [];
-        
-        recorder.onaudioprocess = function(event){
-          const samples = event.inputBuffer.getChannelData(0);
-          const PCM16iSamples = [];
+      recorder.onaudioprocess = function(event) {
+        const samples = event.inputBuffer.getChannelData(0);
+        const PCM16iSamples = samples.map(sample => {
+          let val = Math.floor(32767 * sample);
+          return Math.min(32767, Math.max(-32768, val));
+        });
 
-          for (let i = 0; i < samples.length; i++)
-          {
-            let val = Math.floor(32767 * samples[i]);
-            val = Math.min(32767, val);
-            val = Math.max(-32768, val);
-          
-            PCM16iSamples.push(val);
-          }
+        // Push audio to websocket
+        const int16Array = new Int16Array(PCM16iSamples);
+        const blob = new Blob([int16Array], { type: 'application/octet-stream' });
+        ws.send(blob);
+      };
 
-          // Push audio to websocket
-          const int16Array = new Int16Array(PCM16iSamples);
-          const blob = new Blob([int16Array], { type: 'application/octet-stream' })
-          ws.send(blob);
-
-        };
-
-    }, function(error){
-        alert('Error capturing audio.');
+    }, function(error) {
+      alert('Error capturing audio.');
     });
-  }
-  else
-  {
+  } else {
     alert('getUserMedia not supported in this browser.');
   }
 
   // start recording
+  const startButton = document.getElementById('startButton');
   startButton.addEventListener('click', function() {
-    volume.connect(recorder);
-    recorder.connect(audioContext.destination);
-    ws.send(sampleRate);
-  })
-
+    if (!startButton.classList.contains('listening')) {
+      volume.connect(recorder);
+      recorder.connect(audioContext.destination);
+      ws.send(sampleRate);
+      startButton.classList.add('listening');
+      startButton.textContent = 'Listening...';
+    }
+  });
   </script>
 </body>
 </html>
\ No newline at end of file
diff --git a/examples/web/streaming_server.py b/examples/web/streaming_server.py
index c386bbe..449d251 100644
--- a/examples/web/streaming_server.py
+++ b/examples/web/streaming_server.py
@@ -26,12 +26,16 @@ import numpy as np
 from openwakeword import Model
 import resampy
 import argparse
+import json
 
 # Define websocket handler
 async def websocket_handler(request):
     ws = web.WebSocketResponse()
     await ws.prepare(request)
 
+    # Send loaded models
+    await ws.send_str(json.dumps({"loaded_models": list(owwModel.models.keys())}))
+
     # Start listening for websocket messages
     async for msg in ws:
         # Get the sample rate of the microphone from the browser
@@ -61,7 +65,7 @@ async def websocket_handler(request):
                     activations.append(key)
 
             if activations != []:
-                await ws.send_str(str(activations))
+                await ws.send_str(json.dumps({"activations": activations}))
 
     return ws
 
