diff --git a/.gitignore b/.gitignore index 1e20d39..f074964 100644 --- a/.gitignore +++ b/.gitignore @@ -142,3 +142,4 @@ examples/ # archive files archive/ +node_modules/ diff --git a/web/.gitignore b/web/.gitignore new file mode 100644 index 0000000..b2decfd --- /dev/null +++ b/web/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +models/ +*.onnx diff --git a/web/LICENSE b/web/LICENSE new file mode 100644 index 0000000..cbd5c75 --- /dev/null +++ b/web/LICENSE @@ -0,0 +1,199 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/web/README.md b/web/README.md new file mode 100644 index 0000000..8784325 --- /dev/null +++ b/web/README.md @@ -0,0 +1,269 @@ +# openWakeWord — native browser port + +A native **JavaScript / browser** port of [openWakeWord](../README.md). Wake word +detection runs **fully client-side** in the browser using +[ONNX Runtime Web](https://onnxruntime.ai/docs/get-started/with-javascript/web.html) — +no Python server, no websocket streaming. + +It loads the exact same ONNX models as the Python package (melspectrogram, +speech embedding, and the pre-trained wake word models) and faithfully +reimplements the streaming feature pipeline (`AudioFeatures`) and prediction +logic (`Model.predict`) in JavaScript. + +> Looking for the server-based approach instead? See +> `[examples/web](../examples/web)` for streaming microphone audio over a +> websocket into openWakeWord running in a Python backend. + + + +## Pipeline + +``` +mic (16 kHz, 16-bit PCM) + └─ melspectrogram.onnx → mel features (32 bins), transformed x/10 + 2 + └─ embedding_model.onnx → 96-dim speech embeddings (76-frame windows, step 8) + └─ .onnx → detection score 0..1 +``` + + + +## Quick start + +```bash +cd web +npm install # installs onnxruntime-web +npm run download-models # fetches the ONNX models into ./models +npm run serve # or any static file server +# open http://localhost:8080/demo/ +``` + +Then open the page, click **Start Listening**, and say *"hey jarvis"*, +*"alexa"*, *"hey mycroft"*, or *"hey rhasspy"*. + +> The demo loads `onnxruntime-web` from a CDN via an +> [import map](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/script/type/importmap), +> so it works without a bundler. A static server is still required because ES +> modules and the AudioWorklet cannot be loaded from `file://`. + + + +### Two hard requirements (browser rules) + +- **Must be served from** `http://localhost` **or** `https://`**.** ES modules, +`AudioWorklet`, and microphone access all require a real, secure origin, so +opening the file directly via `file://` will not work. +- **Model files must be same-origin** (or served with CORS headers). The +upstream GitHub release assets do **not** send `Access-Control-Allow-Origin`, +so you cannot point `baseUrl` directly at the GitHub release — host the +`.onnx` files yourself (which is what `npm run download-models` sets up). + + + +## Library usage + +```bash +npm install openwakeword-web +``` + +```js +import { OpenWakeWord } from "openwakeword-web"; +import { Microphone } from "openwakeword-web/microphone"; + +const oww = await OpenWakeWord.create({ + baseUrl: "./models/", + wakewordModels: ["hey_jarvis", "alexa"], // or omit for all pre-trained models + threshold: 0.5, + onDetection: ({ label, score }) => { + console.log(`Wake word detected: ${label} (score ${score.toFixed(2)})`); + }, +}); + +const mic = new Microphone(async (frame) => { + // frame is an Int16Array of 1280 samples (80 ms @ 16 kHz) + await oww.predict(frame); // onDetection fires automatically when threshold is met +}); + +await mic.start(); +``` + +`predict()` accepts any `Int16Array` of 16 kHz PCM audio (ideally multiples of +1280 samples) and returns a `{ label: score }` map, matching the Python API. +The `onDetection` callback fires inside `predict()` for every label whose score +meets `threshold` — so you don't have to poll the returned scores yourself. + +Both `threshold` and `onDetection` can be updated at runtime: + +```js +oww.threshold = 0.75; +oww.onDetection = ({ label }) => triggerAssistant(label); +``` + +If you prefer the polling style, simply omit `onDetection` and inspect the +scores returned by `predict()` directly: + +```js +const scores = await oww.predict(frame); +if (scores["hey_jarvis"] >= 0.5) console.log("detected hey jarvis!"); +``` + +> **Bundlers:** `Microphone` loads its AudioWorklet via +> `new URL("./mic-worklet.js", import.meta.url)`, which Vite and webpack 5 +> handle automatically. If your bundler does not emit the worklet asset, copy +> `mic-worklet.js` somewhere same-origin and pass it explicitly: +> `new Microphone(onFrame, { workletUrl: "/mic-worklet.js" })`. + +> TypeScript type definitions (`.d.ts`) ship with the package, so the API is +> fully typed out of the box. + + + +### Usage in React (Vite) + +Place the `.onnx` model files in `public/assets/models/` so Vite serves them +as static assets, then use a `useEffect` hook to own the lifecycle: + +```tsx +// src/hooks/useWakeWord.ts +import { useEffect, useRef } from "react"; +import { OpenWakeWord, configureOrt } from "openwakeword-web"; +import { Microphone } from "openwakeword-web/microphone"; + +export function useWakeWord( + onDetection: (label: string, score: number) => void, + wakewordModels = ["hey_jarvis", "alexa"], + threshold = 0.5, +) { + // Keep a stable ref so the callback never causes re-initialisation. + const onDetectionRef = useRef(onDetection); + onDetectionRef.current = onDetection; + + useEffect(() => { + let oww: OpenWakeWord | null = null; + let mic: Microphone | null = null; + let cancelled = false; + + configureOrt({ numThreads: 1 }); // avoid COOP/COEP requirement + + OpenWakeWord.create({ + baseUrl: "/assets/models/", + wakewordModels, + threshold, + onDetection: ({ label, score }) => onDetectionRef.current(label, score), + }).then((instance) => { + if (cancelled) return; + oww = instance; + mic = new Microphone(async (frame) => { + await oww!.predict(frame); + }); + mic.start(); + }); + + return () => { + cancelled = true; + mic?.stop(); + oww?.reset(); + }; + }, [threshold, ...wakewordModels]); // re-init if config changes +} +``` + +```tsx +// src/App.tsx +import { useCallback } from "react"; +import { useWakeWord } from "./hooks/useWakeWord"; + +export default function App() { + const handleDetection = useCallback((label: string, score: number) => { + console.log(`Detected: ${label} (${score.toFixed(2)})`); + }, []); + + useWakeWord(handleDetection, ["hey_jarvis", "alexa"], 0.5); + + return
Listening for wake words…
; +} +``` + +**Model files** — copy the `.onnx` files into `public/assets/models/`: + +``` +public/ + assets/ + models/ + melspectrogram.onnx + embedding_model.onnx + hey_jarvis_v0.1.onnx + alexa_v0.1.onnx +``` + +You can copy them from the `web/models/` directory after running +`npm run download-models` in this package, or point `baseUrl` at any +same-origin path that serves the files with correct CORS headers. + + + +### Custom models + +```js +const oww = await OpenWakeWord.create({ + wakewordModels: [ + "alexa", // pre-trained, by name + { name: "my_word", url: "/models/my_word.onnx" }, // your own trained model + ], +}); +``` + + + +### Using your own ONNX runtime / wasm hosting + +```js +import { configureOrt } from "./src/openwakeword.js"; +configureOrt({ wasmPaths: "/ort/", numThreads: 1 }); +``` + +Multi-threaded wasm requires the page to be cross-origin isolated +(`COOP`/`COEP` headers). The demo uses `numThreads: 1` to avoid that +requirement. + +## Layout + + +| Path | Purpose | +| ----------------------------- | ------------------------------------------------------- | +| `src/openwakeword.js` | Main `OpenWakeWord` class — model loading + `predict()` | +| `src/audio-features.js` | Streaming melspectrogram + embedding pipeline | +| `src/models.js` | Pre-trained model registry + class mappings | +| `src/microphone.js` | Mic capture helper (16 kHz `AudioContext` + worklet) | +| `src/mic-worklet.js` | AudioWorklet: float → 16-bit PCM framing | +| `demo/index.html` | Live detection demo UI (uses the `src/` modules) | +| `scripts/download-models.mjs` | Downloads the ONNX models from the GitHub release | +| `test/verify.mjs` | Node script that runs the port on the repo's test clips | + + + + +## Verification + +`test/verify.mjs` streams the repository's test clips +(`tests/data/alexa_test.wav`, `tests/data/hey_mycroft_test.wav`) through the +port under Node and checks the correct wake word fires: + +```bash +cd web +npm test +# alexa_test.wav → alexa = 1.000 PASS +# hey_mycroft_test.wav → hey_mycroft = 1.000 PASS +``` + + + +## Browser support + +Requires `AudioWorklet`, ES modules, and `AudioContext({ sampleRate: 16000 })` +— supported by current Chrome, Edge, Firefox, and Safari. A secure context +(`https://` or `localhost`) is required for microphone access. + +## License + +Apache-2.0, same as openWakeWord. The bundled model files are downloaded from +the upstream openWakeWord releases and retain their original licenses. \ No newline at end of file diff --git a/web/demo/index.html b/web/demo/index.html new file mode 100644 index 0000000..86a5d9c --- /dev/null +++ b/web/demo/index.html @@ -0,0 +1,262 @@ + + + + + + openWakeWord — native browser demo + + + + + + + +
+

openWakeWord

+

Native browser port — wake word detection running fully client-side with ONNX Runtime Web.

+ +
+
+ + Initializing… +
+
+ Threshold: + +
+
+ +
+
+

Try saying "hey jarvis", "alexa", "hey mycroft", or "hey rhasspy".

+
+
+ +
+ + + + diff --git a/web/package-lock.json b/web/package-lock.json new file mode 100644 index 0000000..a1986b6 --- /dev/null +++ b/web/package-lock.json @@ -0,0 +1,158 @@ +{ + "name": "openwakeword-web", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "openwakeword-web", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "onnxruntime-web": "^1.27.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", + "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", + "license": "BSD-3-Clause" + }, + "node_modules/@types/node": { + "version": "26.0.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.0.1.tgz", + "integrity": "sha512-fc3KiUoBt6kie0N9bIW3E47vZsuaMf0PM2AaUpLCLT0s/LvX1nxAim6Fc049cNxODPpGm6qRAuUOB86SkRuPQw==", + "license": "MIT", + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/onnxruntime-common": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.27.0.tgz", + "integrity": "sha512-3KxL5wIVqa8Ex08jxSzncm9CMgw8CjOFyOQ7SxvG9o0cVLlhTNKXyIQuTbtX4tGPJEf73OER2xrjt4HJSBL4ow==", + "license": "MIT" + }, + "node_modules/onnxruntime-web": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.27.0.tgz", + "integrity": "sha512-ogDLsqIozHZwifPuN37OproAo0byX6t43/bP8GzeZWBWD6MOGExswFAx3up4NS/vvWBOg2u2PXomDt3rMmdQSg==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^25.1.24", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.27.0", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, + "node_modules/protobufjs": { + "version": "7.6.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.4.tgz", + "integrity": "sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/undici-types": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "license": "MIT" + } + } +} diff --git a/web/package.json b/web/package.json new file mode 100644 index 0000000..078bf8c --- /dev/null +++ b/web/package.json @@ -0,0 +1,65 @@ +{ + "name": "openwakeword-web", + "version": "0.1.0", + "description": "Native browser (JavaScript) port of openWakeWord. Runs wake word detection fully client-side using ONNX Runtime Web.", + "type": "module", + "main": "src/openwakeword.js", + "types": "src/openwakeword.d.ts", + "exports": { + ".": { + "types": "./src/openwakeword.d.ts", + "import": "./src/openwakeword.js" + }, + "./microphone": { + "types": "./src/microphone.d.ts", + "import": "./src/microphone.js" + }, + "./audio-features": { + "types": "./src/audio-features.d.ts", + "import": "./src/audio-features.js" + }, + "./models": { + "types": "./src/models.d.ts", + "import": "./src/models.js" + } + }, + "files": [ + "src", + "README.md", + "LICENSE" + ], + "scripts": { + "download-models": "node ./scripts/download-models.mjs", + "serve": "python3 -m http.server 8080", + "test": "node ./test/verify.mjs" + }, + "keywords": [ + "wakeword", + "wake-word", + "openwakeword", + "speech", + "voice", + "onnx", + "onnxruntime-web", + "browser", + "webaudio", + "audioworklet" + ], + "author": "Kousthub Raja (https://github.com/kousthubraja)", + "license": "Apache-2.0", + "homepage": "https://github.com/dscripka/openWakeWord/tree/main/web#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/dscripka/openWakeWord.git", + "directory": "web" + }, + "bugs": { + "url": "https://github.com/dscripka/openWakeWord/issues" + }, + "engines": { + "node": ">=18" + }, + "dependencies": { + "onnxruntime-web": "^1.27.0" + } +} diff --git a/web/scripts/download-models.mjs b/web/scripts/download-models.mjs new file mode 100644 index 0000000..2a7e45c --- /dev/null +++ b/web/scripts/download-models.mjs @@ -0,0 +1,59 @@ +#!/usr/bin/env node +// Downloads the ONNX versions of the openWakeWord models into ./models so the +// browser port can run fully client-side. These are the exact same model files +// used by the Python package (the .onnx assets from the v0.5.1 GitHub release). + +import { mkdir, writeFile, access } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const BASE = + "https://github.com/dscripka/openWakeWord/releases/download/v0.5.1"; + +// Feature models (always required) + the pre-trained wake word models. +const MODELS = [ + "melspectrogram.onnx", + "embedding_model.onnx", + "silero_vad.onnx", + "alexa_v0.1.onnx", + "hey_mycroft_v0.1.onnx", + "hey_jarvis_v0.1.onnx", + "hey_rhasspy_v0.1.onnx", + "timer_v0.1.onnx", + "weather_v0.1.onnx", +]; + +// Models live in web/models; this script lives in web/scripts. +const outDir = join(dirname(fileURLToPath(import.meta.url)), "..", "models"); + +async function exists(p) { + try { + await access(p); + return true; + } catch { + return false; + } +} + +async function download(name) { + const dest = join(outDir, name); + if (await exists(dest)) { + console.log(`✓ ${name} (already present)`); + return; + } + const url = `${BASE}/${name}`; + process.stdout.write(`↓ ${name} ... `); + const res = await fetch(url, { redirect: "follow" }); + if (!res.ok) { + throw new Error(`failed to download ${url} (HTTP ${res.status})`); + } + const buf = Buffer.from(await res.arrayBuffer()); + await writeFile(dest, buf); + console.log(`done (${(buf.length / 1024).toFixed(0)} KB)`); +} + +await mkdir(outDir, { recursive: true }); +for (const m of MODELS) { + await download(m); +} +console.log(`\nModels saved to ${outDir}`); diff --git a/web/src/audio-features.d.ts b/web/src/audio-features.d.ts new file mode 100644 index 0000000..18d0946 --- /dev/null +++ b/web/src/audio-features.d.ts @@ -0,0 +1,40 @@ +import type { InferenceSession } from "onnxruntime-web"; + +export const MEL_BINS: number; +export const EMBED_DIM: number; +export const WINDOW_SIZE: number; +export const STEP_SIZE: number; +export const CHUNK: number; + +/** Tensor-ready features: a flat Float32Array plus its dimensions. */ +export interface Features { + data: Float32Array; + dims: number[]; +} + +/** + * Streaming melspectrogram + speech-embedding feature pipeline. Faithful port + * of `openwakeword.utils.AudioFeatures`. All audio is 16-bit PCM @ 16 kHz. + */ +export class AudioFeatures { + constructor( + melspecSession: InferenceSession, + embeddingSession: InferenceSession, + opts?: { sampleRate?: number } + ); + + /** Reset all internal streaming buffers. Call `warmup()` afterwards. */ + reset(skipWarmup?: boolean): void; + + /** Seed the feature buffer with embeddings of ~4 s of audio. Await once after construction/reset. */ + warmup(): Promise; + + /** + * Feed 16-bit PCM @ 16 kHz audio (ideally multiples of 1280 samples). + * @returns the number of samples processed this call. + */ + streamingFeatures(x: Int16Array): Promise; + + /** Most recent feature frames as a tensor-ready `{ data, dims }`. */ + getFeatures(nFeatureFrames?: number, startNdx?: number): Features; +} diff --git a/web/src/audio-features.js b/web/src/audio-features.js new file mode 100644 index 0000000..f2e7f47 --- /dev/null +++ b/web/src/audio-features.js @@ -0,0 +1,249 @@ +import * as ort from "onnxruntime-web"; + +// Faithful JavaScript port of `openwakeword.utils.AudioFeatures`. +// +// Implements the streaming melspectrogram + Google speech_embedding feature +// pipeline. All audio is 16-bit PCM @ 16 kHz, exactly like the Python library. +// +// raw int16 audio -> melspectrogram model -> (frames x 32) mel features +// -> embedding model -> (frames x 96) embeddings +// +// The embeddings are what the wake word models consume. + +const MEL_BINS = 32; +const EMBED_DIM = 96; +const WINDOW_SIZE = 76; // mel frames per embedding window +const STEP_SIZE = 8; // mel frames produced per 1280-sample (80 ms) chunk +const CHUNK = 1280; // samples per processing step (80 ms @ 16 kHz) + +export class AudioFeatures { + /** + * @param {ort.InferenceSession} melspecSession + * @param {ort.InferenceSession} embeddingSession + */ + constructor(melspecSession, embeddingSession, { sampleRate = 16000 } = {}) { + this.melspecSession = melspecSession; + this.embeddingSession = embeddingSession; + this.sampleRate = sampleRate; + + this.melspecInputName = melspecSession.inputNames[0]; // "input" + this.embeddingInputName = embeddingSession.inputNames[0]; // "input_1" + + this.rawDataMaxLen = sampleRate * 10; + this.melspectrogramMaxLen = 10 * 97; // ~10 s of mel frames + this.featureBufferMaxLen = 120; // ~10 s of embedding history + + this.reset(/* skipWarmup */ true); + } + + /** Reset all internal streaming buffers. */ + reset(skipWarmup = false) { + this.rawDataBuffer = []; // numbers (int16 samples) + this.rawDataRemainder = new Int16Array(0); + this.accumulatedSamples = 0; + // Mel buffer is seeded with ones, matching np.ones((76, 32)). + this.melBuffer = []; + for (let i = 0; i < WINDOW_SIZE; i++) { + this.melBuffer.push(new Float32Array(MEL_BINS).fill(1)); + } + // Feature buffer is seeded later by warmup() (needs async model calls). + this.featureBuffer = []; + if (!skipWarmup) { + // synchronous reset cannot run the models; callers should await warmup() + } + } + + /** + * Seed the feature buffer with the embeddings of 4 s of (random) audio, as + * the Python implementation does on init/reset. Must be awaited once after + * construction (and after reset()). + */ + async warmup() { + const audio = new Int16Array(this.sampleRate * 4); + for (let i = 0; i < audio.length; i++) { + audio[i] = Math.floor(Math.random() * 2000 - 1000); + } + this.featureBuffer = await this._getEmbeddings(audio); + } + + // --- melspectrogram ------------------------------------------------------- + + /** + * Compute the melspectrogram of int16 audio. + * Returns an array of Float32Array rows, shape (frames, 32), with the + * `x / 10 + 2` transform applied (matches the Python default). + * @param {Int16Array} int16 + */ + async _getMelspectrogram(int16) { + const x = Float32Array.from(int16); // int16 magnitudes as float (NOT normalized) + const tensor = new ort.Tensor("float32", x, [1, x.length]); + const out = await this.melspecSession.run({ [this.melspecInputName]: tensor }); + const o = out[this.melspecSession.outputNames[0]]; + const dims = o.dims; + const bins = dims[dims.length - 1]; + const frames = dims[dims.length - 2]; + const data = o.data; + const rows = []; + for (let f = 0; f < frames; f++) { + const row = new Float32Array(bins); + const base = f * bins; + for (let b = 0; b < bins; b++) { + row[b] = data[base + b] / 10 + 2; + } + rows.push(row); + } + return rows; + } + + // --- embeddings ----------------------------------------------------------- + + /** + * Run the embedding model over a batch of 76x32 mel windows. + * @param {Float32Array[][]} windows array of windows, each window is 76 rows of 32 + * @returns {Promise} array of 96-dim embedding rows + */ + async _embedWindows(windows) { + const n = windows.length; + if (n === 0) return []; + const data = new Float32Array(n * WINDOW_SIZE * MEL_BINS); + let p = 0; + for (const win of windows) { + for (let r = 0; r < WINDOW_SIZE; r++) { + data.set(win[r], p); + p += MEL_BINS; + } + } + const tensor = new ort.Tensor("float32", data, [n, WINDOW_SIZE, MEL_BINS, 1]); + const out = await this.embeddingSession.run({ + [this.embeddingInputName]: tensor, + }); + const o = out[this.embeddingSession.outputNames[0]]; + const flat = o.data; // length n * 96 + const rows = []; + for (let i = 0; i < n; i++) { + rows.push(flat.slice(i * EMBED_DIM, i * EMBED_DIM + EMBED_DIM)); + } + return rows; + } + + /** + * Compute embeddings for a whole audio clip (used for warmup / batch use). + * @param {Int16Array} int16 + */ + async _getEmbeddings(int16) { + const spec = await this._getMelspectrogram(int16); + const windows = []; + for (let i = 0; i < spec.length; i += STEP_SIZE) { + const window = spec.slice(i, i + WINDOW_SIZE); + if (window.length === WINDOW_SIZE) windows.push(window); + } + return this._embedWindows(windows); + } + + // --- streaming ------------------------------------------------------------ + + _bufferRawData(int16) { + for (let i = 0; i < int16.length; i++) this.rawDataBuffer.push(int16[i]); + if (this.rawDataBuffer.length > this.rawDataMaxLen) { + this.rawDataBuffer = this.rawDataBuffer.slice(-this.rawDataMaxLen); + } + } + + async _streamingMelspectrogram(nSamples) { + if (this.rawDataBuffer.length < 400) { + throw new Error( + "The number of input frames must be at least 400 samples @ 16khz (25 ms)!" + ); + } + const start = Math.max(0, this.rawDataBuffer.length - (nSamples + 160 * 3)); + const tail = Int16Array.from(this.rawDataBuffer.slice(start)); + const rows = await this._getMelspectrogram(tail); + for (const row of rows) this.melBuffer.push(row); + if (this.melBuffer.length > this.melspectrogramMaxLen) { + this.melBuffer = this.melBuffer.slice(-this.melspectrogramMaxLen); + } + } + + /** + * Streaming feature extraction. Feed it 16-bit PCM @ 16 kHz audio frames + * (ideally multiples of 1280 samples / 80 ms). + * @param {Int16Array} x + * @returns {Promise} number of samples that were processed this call + */ + async streamingFeatures(x) { + let processedSamples = 0; + + if (this.rawDataRemainder.length !== 0) { + const merged = new Int16Array(this.rawDataRemainder.length + x.length); + merged.set(this.rawDataRemainder, 0); + merged.set(x, this.rawDataRemainder.length); + x = merged; + this.rawDataRemainder = new Int16Array(0); + } + + if (this.accumulatedSamples + x.length >= CHUNK) { + const remainder = (this.accumulatedSamples + x.length) % CHUNK; + if (remainder !== 0) { + const xEven = x.subarray(0, x.length - remainder); + this._bufferRawData(xEven); + this.accumulatedSamples += xEven.length; + this.rawDataRemainder = x.slice(x.length - remainder); + } else { + this._bufferRawData(x); + this.accumulatedSamples += x.length; + this.rawDataRemainder = new Int16Array(0); + } + } else { + this.accumulatedSamples += x.length; + this._bufferRawData(x); + } + + if (this.accumulatedSamples >= CHUNK && this.accumulatedSamples % CHUNK === 0) { + await this._streamingMelspectrogram(this.accumulatedSamples); + + // Compute new embeddings for each newly-available 80 ms chunk. + for (let i = this.accumulatedSamples / CHUNK - 1; i >= 0; i--) { + const ndx = -STEP_SIZE * i === 0 ? this.melBuffer.length : -STEP_SIZE * i; + const endAbs = ndx < 0 ? this.melBuffer.length + ndx : ndx; + const startAbs = endAbs - WINDOW_SIZE; + if (startAbs >= 0) { + const window = this.melBuffer.slice(startAbs, endAbs); + const [embedding] = await this._embedWindows([window]); + this.featureBuffer.push(embedding); + } + } + + processedSamples = this.accumulatedSamples; + this.accumulatedSamples = 0; + } + + if (this.featureBuffer.length > this.featureBufferMaxLen) { + this.featureBuffer = this.featureBuffer.slice(-this.featureBufferMaxLen); + } + + return processedSamples !== 0 ? processedSamples : this.accumulatedSamples; + } + + /** + * Return the most recent feature frames as a tensor-ready {data, dims}. + * Mirrors `AudioFeatures.get_features`. + * @param {number} nFeatureFrames + * @param {number} startNdx negative index into the feature buffer, or -1 + */ + getFeatures(nFeatureFrames = 16, startNdx = -1) { + let frames; + if (startNdx !== -1) { + const end = + startNdx + nFeatureFrames === 0 ? undefined : startNdx + nFeatureFrames; + frames = this.featureBuffer.slice(startNdx, end); + } else { + frames = this.featureBuffer.slice(-nFeatureFrames); + } + const n = frames.length; + const data = new Float32Array(n * EMBED_DIM); + for (let i = 0; i < n; i++) data.set(frames[i], i * EMBED_DIM); + return { data, dims: [1, n, EMBED_DIM] }; + } +} + +export { MEL_BINS, EMBED_DIM, WINDOW_SIZE, STEP_SIZE, CHUNK }; diff --git a/web/src/mic-worklet.js b/web/src/mic-worklet.js new file mode 100644 index 0000000..c55cce5 --- /dev/null +++ b/web/src/mic-worklet.js @@ -0,0 +1,57 @@ +// AudioWorklet processor that converts the microphone stream into 16-bit PCM +// frames of 1280 samples (80 ms @ 16 kHz) and posts them to the main thread. +// +// It resamples from the AudioContext's native rate (the global `sampleRate` +// inside the worklet scope) down/up to 16 kHz using linear interpolation, so +// it works even when the browser ignores the requested 16 kHz context rate. + +const TARGET_RATE = 16000; +const FRAME = 1280; + +class PCMWorklet extends AudioWorkletProcessor { + constructor() { + super(); + this._ratio = sampleRate / TARGET_RATE; // input samples per output sample + this._buf = new Int16Array(FRAME); + this._n = 0; + this._tail = new Float32Array(0); // leftover input samples between blocks + this._frac = 0; // fractional read position within the current data buffer + } + + process(inputs) { + const channel = inputs[0]?.[0]; + if (!channel) return true; + + // Prepend any leftover samples needed for cross-block interpolation. + let data = channel; + if (this._tail.length) { + data = new Float32Array(this._tail.length + channel.length); + data.set(this._tail, 0); + data.set(channel, this._tail.length); + } + + const ratio = this._ratio; + let t = this._frac; + while (Math.floor(t) + 1 < data.length) { + const i = Math.floor(t); + const frac = t - i; + const s = data[i] + (data[i + 1] - data[i]) * frac; // linear interp + let v = Math.floor(32767 * s); + if (v > 32767) v = 32767; + else if (v < -32768) v = -32768; + this._buf[this._n++] = v; + if (this._n === FRAME) { + this.port.postMessage(this._buf.slice()); + this._n = 0; + } + t += ratio; + } + + const keepFrom = Math.floor(t); + this._tail = data.slice(keepFrom); + this._frac = t - keepFrom; + return true; + } +} + +registerProcessor("pcm-worklet", PCMWorklet); diff --git a/web/src/microphone.d.ts b/web/src/microphone.d.ts new file mode 100644 index 0000000..99db4dc --- /dev/null +++ b/web/src/microphone.d.ts @@ -0,0 +1,25 @@ +export interface MicrophoneOptions { + /** + * URL of the AudioWorklet module (`mic-worklet.js`). Defaults to the file + * shipped next to this module. Override it if your bundler does not emit the + * worklet asset automatically (point it at a copy served same-origin). + */ + workletUrl?: string; +} + +/** + * Captures microphone audio as 16-bit PCM @ 16 kHz and delivers it in + * 1280-sample (80 ms) frames via the `onFrame` callback. + */ +export class Microphone { + constructor(onFrame: (frame: Int16Array) => void, opts?: MicrophoneOptions); + + /** Request mic access and start delivering frames. Requires a secure context. */ + start(): Promise; + + /** Stop capture and release the microphone. */ + stop(): Promise; + + /** Actual sample rate of the capture context (should be 16000), or null if not started. */ + readonly sampleRate: number | null; +} diff --git a/web/src/microphone.js b/web/src/microphone.js new file mode 100644 index 0000000..2f432de --- /dev/null +++ b/web/src/microphone.js @@ -0,0 +1,74 @@ +// Helper to capture microphone audio as 16-bit PCM @ 16 kHz frames and feed +// them to a callback. Uses an AudioWorklet (see mic-worklet.js). + +export class Microphone { + /** + * @param {(frame: Int16Array) => void} onFrame called with 1280-sample frames + * @param {{workletUrl?: string}} [opts] + */ + constructor(onFrame, opts = {}) { + this.onFrame = onFrame; + this.workletUrl = + opts.workletUrl ?? new URL("./mic-worklet.js", import.meta.url).href; + this.context = null; + this.stream = null; + this.node = null; + this.source = null; + } + + async start() { + if (this.context) return; + + this.stream = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + + // Prefer a 16 kHz context (the worklet also resamples as a fallback in + // case the browser ignores or rejects this hint). + const AudioCtx = globalThis.AudioContext || globalThis.webkitAudioContext; + try { + this.context = new AudioCtx({ sampleRate: 16000 }); + } catch { + this.context = new AudioCtx(); + } + if (this.context.state === "suspended") await this.context.resume(); + await this.context.audioWorklet.addModule(this.workletUrl); + + this.source = this.context.createMediaStreamSource(this.stream); + this.node = new AudioWorkletNode(this.context, "pcm-worklet"); + this.node.port.onmessage = (e) => this.onFrame(e.data); + + this.source.connect(this.node); + // Keep the worklet pulling audio without routing mic to the speakers. + const sink = this.context.createGain(); + sink.gain.value = 0; + this.node.connect(sink); + sink.connect(this.context.destination); + this._sink = sink; + } + + /** Actual sample rate of the capture context (should be 16000). */ + get sampleRate() { + return this.context ? this.context.sampleRate : null; + } + + async stop() { + if (this.node) this.node.port.onmessage = null; + try { + this.source?.disconnect(); + this.node?.disconnect(); + this._sink?.disconnect(); + } catch {} + if (this.stream) this.stream.getTracks().forEach((t) => t.stop()); + if (this.context) await this.context.close(); + this.context = null; + this.stream = null; + this.node = null; + this.source = null; + } +} diff --git a/web/src/models.d.ts b/web/src/models.d.ts new file mode 100644 index 0000000..f0d311d --- /dev/null +++ b/web/src/models.d.ts @@ -0,0 +1,11 @@ +/** Feature models required by every wake word model. */ +export const FEATURE_MODELS: { melspectrogram: string; embedding: string }; + +/** Voice-activity-detection models (not yet used by the browser port). */ +export const VAD_MODELS: { silero_vad: string }; + +/** Pre-trained wake word models, keyed by name -> ONNX filename. */ +export const PRETRAINED_MODELS: Record; + +/** Integer-class -> label mappings for multi-class models (e.g. `timer`). */ +export const MODEL_CLASS_MAPPINGS: Record>; diff --git a/web/src/models.js b/web/src/models.js new file mode 100644 index 0000000..aa2eb79 --- /dev/null +++ b/web/src/models.js @@ -0,0 +1,33 @@ +// Registry of the pre-trained openWakeWord models and their class mappings. +// Mirrors `openwakeword/__init__.py` from the Python package. + +export const FEATURE_MODELS = { + melspectrogram: "melspectrogram.onnx", + embedding: "embedding_model.onnx", +}; + +export const VAD_MODELS = { + silero_vad: "silero_vad.onnx", +}; + +// name -> onnx filename (relative to the models directory / base URL) +export const PRETRAINED_MODELS = { + alexa: "alexa_v0.1.onnx", + hey_mycroft: "hey_mycroft_v0.1.onnx", + hey_jarvis: "hey_jarvis_v0.1.onnx", + hey_rhasspy: "hey_rhasspy_v0.1.onnx", + timer: "timer_v0.1.onnx", + weather: "weather_v0.1.onnx", +}; + +// Integer-class -> label mappings for multi-class models. +export const MODEL_CLASS_MAPPINGS = { + timer: { + 1: "1_minute_timer", + 2: "5_minute_timer", + 3: "10_minute_timer", + 4: "20_minute_timer", + 5: "30_minute_timer", + 6: "1_hour_timer", + }, +}; diff --git a/web/src/openwakeword.d.ts b/web/src/openwakeword.d.ts new file mode 100644 index 0000000..0f942e0 --- /dev/null +++ b/web/src/openwakeword.d.ts @@ -0,0 +1,100 @@ +import { AudioFeatures } from "./audio-features.js"; + +export * from "./models.js"; +export { AudioFeatures } from "./audio-features.js"; + +/** Options for {@link configureOrt}. */ +export interface ConfigureOrtOptions { + /** Base URL/path for the ONNX Runtime Web wasm binaries. */ + wasmPaths?: string; + /** Number of wasm threads. Use 1 to avoid requiring COOP/COEP headers. */ + numThreads?: number; + /** Enable SIMD wasm. */ + simd?: boolean; +} + +/** + * Configure the ONNX Runtime Web environment. Call once before + * {@link OpenWakeWord.create} to point at self-hosted wasm or tweak threading. + */ +export function configureOrt(opts?: ConfigureOrtOptions): void; + +/** A custom wake word model supplied by URL. */ +export interface CustomWakewordModel { + name: string; + url: string; + inputFrames?: number; + classMapping?: Record; +} + +/** Wake word model reference: a pre-trained name, or a custom model by URL. */ +export type WakewordModelRef = string | CustomWakewordModel; + +/** Payload passed to {@link OpenWakeWordOptions.onDetection}. */ +export interface DetectionEvent { + /** The wake word label that was detected. */ + label: string; + /** Detection score in 0..1. */ + score: number; +} + +/** Options for {@link OpenWakeWord.create}. */ +export interface OpenWakeWordOptions { + /** Base URL/path for model files. Default `"./models/"`. */ + baseUrl?: string; + /** + * Wake word models to load. Strings are looked up in the pre-trained registry + * (e.g. `"hey_jarvis"`); objects load a custom model by URL. Defaults to all + * pre-trained models. + */ + wakewordModels?: WakewordModelRef[]; + /** Override the melspectrogram model URL. */ + melspectrogramUrl?: string; + /** Override the embedding model URL. */ + embeddingUrl?: string; + /** ONNX Runtime execution providers. Default `["wasm"]`. */ + executionProviders?: string[]; + /** Options forwarded to {@link configureOrt}. */ + ort?: ConfigureOrtOptions; + /** + * Score threshold for triggering {@link onDetection}. Default `0.5`. + * Also stored as `oww.threshold` so it can be changed at runtime. + */ + threshold?: number; + /** + * Called from within {@link OpenWakeWord.predict} whenever a label's score + * meets or exceeds {@link threshold}. May be called multiple times per + * `predict()` invocation if several labels fire simultaneously. + */ + onDetection?: (event: DetectionEvent) => void; +} + +/** + * Native browser port of `openwakeword.Model`. Runs the full melspectrogram -> + * embedding -> wake word pipeline client-side using ONNX Runtime Web. + */ +export class OpenWakeWord { + features: AudioFeatures; + /** Detection score threshold. Can be updated at runtime. Default `0.5`. */ + threshold: number; + /** Callback fired on detection. Can be replaced at runtime. */ + onDetection: ((event: DetectionEvent) => void) | null; + + /** Create and initialise a model. */ + static create(opts?: OpenWakeWordOptions): Promise; + + /** Names of the loaded wake word models. */ + readonly modelNames: string[]; + + /** Reset all streaming/prediction state. */ + reset(): Promise; + + /** + * Predict wake word scores for a frame of 16-bit PCM @ 16 kHz audio (ideally + * multiples of 1280 samples / 80 ms). + * @returns a `{ label: score }` map, score in 0..1. + */ + predict(x: Int16Array): Promise>; +} + +export default OpenWakeWord; diff --git a/web/src/openwakeword.js b/web/src/openwakeword.js new file mode 100644 index 0000000..e81dc65 --- /dev/null +++ b/web/src/openwakeword.js @@ -0,0 +1,239 @@ +import * as ort from "onnxruntime-web"; +import { AudioFeatures, CHUNK } from "./audio-features.js"; +import { + FEATURE_MODELS, + PRETRAINED_MODELS, + MODEL_CLASS_MAPPINGS, +} from "./models.js"; + +const DEFAULT_INPUT_FRAMES = 16; // openWakeWord models use 16 feature frames + +/** + * Configure the ONNX Runtime Web environment. Call once before creating a model + * if you want to point at self-hosted wasm binaries or tweak threading. + * @param {{wasmPaths?: string, numThreads?: number, simd?: boolean}} opts + */ +export function configureOrt(opts = {}) { + if (opts.wasmPaths !== undefined) ort.env.wasm.wasmPaths = opts.wasmPaths; + if (opts.numThreads !== undefined) ort.env.wasm.numThreads = opts.numThreads; + if (opts.simd !== undefined) ort.env.wasm.simd = opts.simd; +} + +function readShapeDim(session, which, idx) { + // Best-effort read of an input/output dimension across ort-web versions. + const meta = + which === "input" + ? session.inputMetadata?.[0] + : session.outputMetadata?.[0]; + const shape = meta?.shape ?? meta?.dimensions; + const v = shape?.[idx]; + return typeof v === "number" && v > 0 ? v : null; +} + +/** + * Native browser port of `openwakeword.Model`. + * + * Runs the full melspectrogram -> embedding -> wake word pipeline client-side + * using ONNX Runtime Web. No server required. + */ +export class OpenWakeWord { + constructor() { + this.models = {}; // name -> { session, inputName, inputFrames, outputClasses, classMapping } + this.features = null; + this.predictionBuffer = {}; // label -> number[] (max 30) + this.threshold = 0.5; + this.onDetection = null; + } + + /** + * Create and initialise a model. + * + * @param {object} opts + * @param {string} [opts.baseUrl="./models/"] Base URL/path for model files. + * @param {Array} [opts.wakewordModels] + * Wake word models to load. Strings are looked up in the pre-trained + * registry (e.g. "hey_jarvis"); objects allow custom models by URL. + * Defaults to all pre-trained models. + * @param {string} [opts.melspectrogramUrl] Override the melspectrogram model URL. + * @param {string} [opts.embeddingUrl] Override the embedding model URL. + * @param {string[]} [opts.executionProviders=["wasm"]] ORT execution providers. + * @param {object} [opts.ort] Options forwarded to {@link configureOrt}. + * @returns {Promise} + */ + static async create(opts = {}) { + const { + baseUrl = "./models/", + wakewordModels = Object.keys(PRETRAINED_MODELS), + executionProviders = ["wasm"], + ort: ortOpts, + threshold = 0.5, + onDetection = null, + } = opts; + + if (ortOpts) configureOrt(ortOpts); + + const join = (file) => + /^https?:|^\.|^\//.test(file) ? file : baseUrl + file; + const sessOpts = { executionProviders }; + + const melspectrogramUrl = opts.melspectrogramUrl + ? opts.melspectrogramUrl + : join(FEATURE_MODELS.melspectrogram); + const embeddingUrl = opts.embeddingUrl + ? opts.embeddingUrl + : join(FEATURE_MODELS.embedding); + + const self = new OpenWakeWord(); + + // Load feature models + create the streaming feature extractor. + const [melspecSession, embeddingSession] = await Promise.all([ + ort.InferenceSession.create(melspectrogramUrl, sessOpts), + ort.InferenceSession.create(embeddingUrl, sessOpts), + ]); + self.features = new AudioFeatures(melspecSession, embeddingSession); + + // Load wake word models. + for (const entry of wakewordModels) { + let name, url, inputFrames, classMapping; + if (typeof entry === "string") { + name = entry; + const file = PRETRAINED_MODELS[entry] || entry; + url = join(file); + classMapping = MODEL_CLASS_MAPPINGS[entry]; + } else { + name = entry.name; + url = /^https?:|^\.|^\//.test(entry.url) ? entry.url : join(entry.url); + inputFrames = entry.inputFrames; + classMapping = entry.classMapping || MODEL_CLASS_MAPPINGS[name]; + } + + const session = await ort.InferenceSession.create(url, sessOpts); + const detectedFrames = readShapeDim(session, "input", 1); + const outputClasses = readShapeDim(session, "output", 1) ?? 1; + self.models[name] = { + session, + inputName: session.inputNames[0], + inputFrames: inputFrames ?? detectedFrames ?? DEFAULT_INPUT_FRAMES, + outputClasses, + classMapping: classMapping || null, + }; + } + + self.threshold = threshold; + self.onDetection = onDetection; + + await self.features.warmup(); + return self; + } + + /** Names of the loaded wake word models. */ + get modelNames() { + return Object.keys(this.models); + } + + /** Reset all streaming/prediction state. */ + async reset() { + this.features.reset(true); + await this.features.warmup(); + this.predictionBuffer = {}; + } + + async _runModel(m, feat) { + const tensor = new ort.Tensor("float32", feat.data, feat.dims); + const out = await m.session.run({ [m.inputName]: tensor }); + const data = out[m.session.outputNames[0]].data; + return Array.from(data); // length = outputClasses (the [0] row) + } + + _pushPrediction(label, value) { + if (!this.predictionBuffer[label]) this.predictionBuffer[label] = []; + this.predictionBuffer[label].push(value); + if (this.predictionBuffer[label].length > 30) { + this.predictionBuffer[label].shift(); + } + } + + /** + * Predict wake word scores for a frame of 16-bit PCM @ 16 kHz audio. + * Ideally pass multiples of 1280 samples (80 ms). + * + * @param {Int16Array} x + * @returns {Promise>} label -> score (0..1) + */ + async predict(x) { + if (!(x instanceof Int16Array)) { + throw new TypeError("Input audio (x) must be an Int16Array of 16 kHz PCM."); + } + + const nPrepared = await this.features.streamingFeatures(x); + const predictions = {}; + + for (const [name, m] of Object.entries(this.models)) { + let prediction; // array of length outputClasses + + if (nPrepared > CHUNK) { + const group = []; + for (let i = Math.floor(nPrepared / CHUNK) - 1; i >= 0; i--) { + const feat = this.features.getFeatures( + m.inputFrames, + -m.inputFrames - i + ); + group.push(await this._runModel(m, feat)); + } + prediction = group.reduce((acc, row) => + acc.map((v, idx) => Math.max(v, row[idx])) + ); + } else if (nPrepared === CHUNK) { + const feat = this.features.getFeatures(m.inputFrames); + prediction = await this._runModel(m, feat); + } else { + // Not enough new samples yet: reuse the previous prediction. + if (m.outputClasses === 1) { + const buf = this.predictionBuffer[name]; + prediction = [buf && buf.length > 0 ? buf[buf.length - 1] : 0]; + } else { + prediction = new Array(m.outputClasses).fill(0); + } + } + + if (m.outputClasses === 1) { + predictions[name] = prediction[0]; + } else if (m.classMapping) { + for (const [intLabel, cls] of Object.entries(m.classMapping)) { + predictions[cls] = prediction[Number.parseInt(intLabel, 10)]; + } + } else { + for (let c = 0; c < m.outputClasses; c++) { + predictions[`${name}_${c}`] = prediction[c]; + } + } + } + + // Zero out predictions for the first 5 frames during model warm-up. + for (const label of Object.keys(predictions)) { + if (!this.predictionBuffer[label] || this.predictionBuffer[label].length < 5) { + predictions[label] = 0; + } + } + + // Update the prediction history buffers. + for (const label of Object.keys(predictions)) { + this._pushPrediction(label, predictions[label]); + } + + // Fire detection callback for any label that meets the threshold. + if (this.onDetection) { + for (const [label, score] of Object.entries(predictions)) { + if (score >= this.threshold) { + this.onDetection({ label, score }); + } + } + } + + return predictions; + } +} + +export { AudioFeatures } from "./audio-features.js"; +export * from "./models.js"; +export default OpenWakeWord; diff --git a/web/test/verify.mjs b/web/test/verify.mjs new file mode 100644 index 0000000..6184d16 --- /dev/null +++ b/web/test/verify.mjs @@ -0,0 +1,87 @@ +// End-to-end verification of the browser port, run under Node with the same +// onnxruntime-web package. Streams real test clips through the model exactly +// like Python's `Model.predict_clip` and checks that the right wake word fires. + +import { readFile } from "node:fs/promises"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { OpenWakeWord, configureOrt } from "../src/openwakeword.js"; + +// This script lives in web/test. The web package root is one level up, and the +// repository root (which holds tests/data/*.wav) is two levels up. +const here = dirname(fileURLToPath(import.meta.url)); +const webRoot = join(here, ".."); +const repo = join(here, "..", ".."); +const modelsDir = join(webRoot, "models") + "/"; + +configureOrt({ numThreads: 1 }); + +// Minimal 16-bit PCM mono WAV reader -> Int16Array of samples. +function readWavInt16(buf) { + const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength); + let off = 12; // skip RIFF header + let dataOffset = -1; + let dataLen = 0; + while (off + 8 <= dv.byteLength) { + const id = String.fromCharCode( + dv.getUint8(off), dv.getUint8(off + 1), dv.getUint8(off + 2), dv.getUint8(off + 3) + ); + const size = dv.getUint32(off + 4, true); + if (id === "data") { + dataOffset = off + 8; + dataLen = size; + break; + } + off += 8 + size + (size % 2); + } + if (dataOffset < 0) throw new Error("no data chunk"); + return new Int16Array(buf.buffer, buf.byteOffset + dataOffset, dataLen / 2); +} + +async function predictClip(oww, samples, { padding = 1, chunk = 1280 } = {}) { + const pad = 16000 * padding; + const data = new Int16Array(pad + samples.length + pad); + data.set(samples, pad); + + const maxScores = {}; + for (let i = 0; i + chunk < data.length; i += chunk) { + const frame = data.slice(i, i + chunk); + const scores = await oww.predict(frame); + for (const [k, v] of Object.entries(scores)) { + maxScores[k] = Math.max(maxScores[k] ?? 0, v); + } + } + return maxScores; +} + +const CASES = [ + { file: "tests/data/alexa_test.wav", expect: "alexa" }, + { file: "tests/data/hey_mycroft_test.wav", expect: "hey_mycroft" }, +]; + +const oww = await OpenWakeWord.create({ + baseUrl: modelsDir, + wakewordModels: ["alexa", "hey_mycroft", "hey_jarvis", "hey_rhasspy"], +}); +console.log("Loaded models:", oww.modelNames.join(", ")); +console.log( + "Input frames per model:", + Object.fromEntries(Object.entries(oww.models).map(([k, v]) => [k, v.inputFrames])) +); + +let allPass = true; +for (const { file, expect } of CASES) { + const buf = await readFile(join(repo, file)); + const samples = readWavInt16(buf); + await oww.reset(); + const scores = await predictClip(oww, samples); + const top = Object.entries(scores).sort((a, b) => b[1] - a[1]); + const pass = scores[expect] >= 0.5 && top[0][0] === expect; + allPass = allPass && pass; + console.log(`\n${file}`); + console.log(" max scores:", Object.fromEntries(top.map(([k, v]) => [k, +v.toFixed(3)]))); + console.log(` expected "${expect}" -> ${pass ? "PASS" : "FAIL"} (score ${(scores[expect] ?? 0).toFixed(3)})`); +} + +console.log(`\n${allPass ? "ALL PASS ✅" : "FAILURES ❌"}`); +process.exit(allPass ? 0 : 1);