akadmin/openwakeword
1
0
Fork 0
mirror of https://github.com/dscripka/openWakeWord.git synced 2026-08-27 18:17:20 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Basic web streaming example [skip ci]

Browse source
This commit is contained in:
David Scripka 2023-11-08 07:48:56 -05:00
parent a2522e29fe
commit 8376848be5
3 changed files with 235 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

15
examples/web/README.md Normal file
View file

@ -0,0 +1,15 @@
# Examples
This folder contains examples of using openWakeWord with web applications.
## Websocket Streaming
As openWakeWord does not have a native Javascript port, using it within a web browswer is best accomplished with websocket streaming of the audio data from the browser to a simple Python application. To install the requirements for this example:
```
pip install aiohttp
```
The `streaming_client.html` page shows a simple implementation of audio capture and streamimng from a microphone and streaming in a browser, and the `streaming_server.py` file is the corresponding websocket server that passes the audio into openWakeWord.
To run the example, execute `python streaming_server.py` (add the `--help` argument to see options) and navigate to `localhost:9000` in your browser.

112
examples/web/streaming_client.html Normal file
View file

@ -0,0 +1,112 @@
<html>
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Websocket Microphone Streaming</title>
</head>
<body>
<h1>Streaming Audio to openWakeWord Using Websockets</h1>
<button id="startButton">Start Recording</button>
<script>
// Create websocket connection
ws = new WebSocket('ws://localhost:9000/ws');
// When the websocket connection is open
ws.onopen = function() {
console.log('WebSocket connection is open');
};
// Get responses from websocket
ws.onmessage = (event) => {
console.log(event.data);
};
// Create microphone capture stream
// Based on the excellent guide here: https://medium.com/@ragymorkos/gettineg-monochannel-16-bit-signed-integer-pcm-audio-samples-from-the-microphone-in-the-browser-8d4abf81164d
navigator.getUserMedia = navigator.getUserMedia ||
navigator.webkitGetUserMedia ||
navigator.mozGetUserMedia ||
navigator.msGetUserMedia;
let audioStream;
let audioContext;
let recorder;
let volume;
let sampleRate
if (navigator.getUserMedia)
{
navigator.getUserMedia({audio: true}, function(stream){
audioStream = stream;
// creates the an instance of audioContext
const context = window.AudioContext || window.webkitAudioContext;
audioContext = new context();
// retrieve the current sample rate of microphone the browser is using and send to Python server
sampleRate = audioContext.sampleRate;
// creates a gain node
volume = audioContext.createGain();
// creates an audio node from the microphone incoming stream
const audioInput = audioContext.createMediaStreamSource(audioStream);
// connect the stream to the gain node
audioInput.connect(volume);
/* From the spec: This value controls how frequently the audioprocess event is
dispatched and how many sample-frames need to be processed each call.
Lower values for buffer size will result in a lower (better) latency.
Higher values will be necessary to avoid audio breakup and glitches */
const bufferSize = 4096;
recorder = (audioContext.createScriptProcessor ||
audioContext.createJavaScriptNode).call(audioContext,
bufferSize,
1,
1);
const leftChannel = [];
recorder.onaudioprocess = function(event){
const samples = event.inputBuffer.getChannelData(0);
const PCM16iSamples = [];
for (let i = 0; i < samples.length; i++)
{
let val = Math.floor(32767 * samples[i]);
val = Math.min(32767, val);
val = Math.max(-32768, val);
PCM16iSamples.push(val);
}
// Push audio to websocket
const int16Array = new Int16Array(PCM16iSamples);
const blob = new Blob([int16Array], { type: 'application/octet-stream' })
ws.send(blob);
};
}, function(error){
alert('Error capturing audio.');
});
}
else
{
alert('getUserMedia not supported in this browser.');
}
// start recording
startButton.addEventListener('click', function() {
volume.connect(recorder);
recorder.connect(audioContext.destination);
ws.send(sampleRate);
})
</script>
</body>
</html>

108
examples/web/streaming_server.py Normal file
View file

@ -0,0 +1,108 @@
# Copyright 2023 David Scripka. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#######################################################################################
# This example scripts runs openWakeWord in a simple web server receiving audio
# from a web page using websockets.
#######################################################################################
# Imports
import aiohttp
from aiohttp import web
import numpy as np
from openwakeword import Model
import resampy
import argparse
# Define websocket handler
async def websocket_handler(request):
ws = web.WebSocketResponse()
await ws.prepare(request)
# Start listening for websocket messages
async for msg in ws:
# Get the sample rate of the microphone from the browser
if msg.type == aiohttp.WSMsgType.TEXT:
sample_rate = int(msg.data)
elif msg.type == aiohttp.WSMsgType.ERROR:
print(f"WebSocket error: {ws.exception()}")
else:
# Get audio data from websocket
audio_bytes = msg.data
# Add extra bytes of silence if needed
if len(msg.data) % 2 == 1:
audio_bytes += (b'\x00')
# Convert audio to correct format and sample rate
data = np.frombuffer(audio_bytes, dtype=np.int16)
if sample_rate != 16000:
data = resampy.resample(data, sample_rate, 16000)
# Get openWakeWord predictions and set to browser client
predictions = owwModel.predict(data)
activations = []
for key in predictions:
if predictions[key] >= 0.5:
activations.append(key)
if activations != []:
await ws.send_str(str(activations))
return ws
# Define static file handler
async def static_file_handler(request):
return web.FileResponse('./streaming_client.html')
app = web.Application()
app.add_routes([web.get('/ws', websocket_handler), web.get('/', static_file_handler)])
if __name__ == '__main__':
# Parse CLI arguments
parser=argparse.ArgumentParser()
parser.add_argument(
"--chunk_size",
help="How much audio (in number of samples) to predict on at once",
type=int,
default=1280,
required=False
)
parser.add_argument(
"--model_path",
help="The path of a specific model to load",
type=str,
default="",
required=False
)
parser.add_argument(
"--inference_framework",
help="The inference framework to use (either 'onnx' or 'tflite'",
type=str,
default='tflite',
required=False
)
args=parser.parse_args()
# Load openWakeWord models
if args.model_path != "":
owwModel = Model(wakeword_models=[args.model_path], inference_framework=args.inference_framework)
else:
owwModel = Model(inference_framework=args.inference_framework)
# Start webapp
web.run_app(app, host='localhost', port=9000)