Update install instructions

Add support for using phonemes as direct input

CHANGELOG.md

@@ -1,9 +1,19 @@
 # Changelog
 
+## 3.2.0
+
+- Refactor as `piper_sample_generator` package
+
+## 3.1.0
+
+- Support MPS acceleration on Apple Silicon
+- Add `--phoneme-input` flag
+
 ## 3.0.0
 
 - Move phonemization to piper 1.3.0 (piper-phonemize is deprecated)
 - Move to PyTorch 2
 - Add support for using Piper voices (`.onnx`) directly
+- Allow multiple `--model` for Piper voices (`.onnx`)
 - Remove silence trimming
 - Remove `min-phoneme-count`

README.md

@@ -6,16 +6,8 @@ Supports normal [Piper voices][piper voices] or a special [generator][] that can
 
 ## Install
 
-Create a virtual environment and install the requirements:
-
 ``` sh
-git clone https://github.com/rhasspy/piper-sample-generator.git
-cd piper-sample-generator/
-
-python3 -m venv .venv
-source .venv/bin/activate
-python3 -m pip install --upgrade pip
-python3 -m pip install -e .
+pip install piper-sample-generator
 ```
 
 ## Piper Voices
@@ -33,7 +25,7 @@ wget -O voices/en_US-lessac-medium.onnx.json 'https://huggingface.co/rhasspy/pip
 Generate a small set of samples with the CLI:
 
 ``` sh
-python3 generate_samples.py 'okay piper.' --model voices/en_US-lessac-medium.onnx --max-samples 10 --output-dir okay_piper/
+python3 -m piper_sample_generator 'okay piper.' --model voices/en_US-lessac-medium.onnx --max-samples 10 --output-dir okay_piper/
 ```
 
 Check the `okay_piper/` directory for 10 WAV files (named `0.wav` to `9.wav`).
@@ -53,7 +45,7 @@ wget -O models/en-us-libritts-high.pt 'https://github.com/rhasspy/piper-sample-g
 Generate a small set of samples with the CLI:
 
 ``` sh
-python3 generate_samples.py 'okay piper.' --model models/en-us-libritts-high.pt --max-samples 10 --output-dir okay_piper/
+python3 -m piper_sample_generator 'okay piper.' --model models/en-us-libritts-high.pt --max-samples 10 --output-dir okay_piper/
 ```
 
 Check the `okay_piper/` directory for 10 WAV files (named `0.wav` to `9.wav`).
@@ -61,7 +53,7 @@ Check the `okay_piper/` directory for 10 WAV files (named `0.wav` to `9.wav`).
 Generation can be much faster and more efficient if you have a GPU available and PyTorch is configured to use it. In this case, increase the batch size:
 
 ``` sh
-python3 generate_samples.py 'okay piper.' --model models/en-us-libritts-high.pt --max-samples 100 --batch-size 10 --output-dir okay_piper/
+python3 -m piper_sample_generator 'okay piper.' --model models/en-us-libritts-high.pt --max-samples 100 --batch-size 10 --output-dir okay_piper/
 ```
 
 On an NVidia 2080 Ti with 11GB, a batch size of 100 was possible (generating approximately 100 samples per second).
@@ -75,14 +67,14 @@ See `--help` for more options, including the `--length-scales` (speaking speeds)
 Once you have samples generated, you can augment them using [audiomentation](https://iver56.github.io/audiomentations/):
 
 ``` sh
-python3 augment.py --sample-rate 22050 okay_piper/ okay_piper_augmented/
+python3 -m piper_sample_generator.augment --sample-rate 22050 okay_piper/ okay_piper_augmented/
 ```
 
 This will do several things to each sample:
 
 1. Randomly decrease the volume
     * The original samples are normalized, so different volume levels are needed
-2. Randomly apply an [impulse response][] using the files in `impulses/`
+2. Randomly apply an [impulse response][] using the files in `piper_sample_generator/impulses/`
     * Change the acoustics of the sample to sound like the speaker was in a room with echo or using a poor quality microphone
 3. Resample to 16Khz for training (e.g., [openWakeWord][])
 

piper_sample_generator/__init__.py

@@ -0,0 +1 @@
+"""Piper sample generator."""

piper_sample_generator/__main__.py

@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 import argparse
+import gc
 import itertools as it
 import json
 import logging
 import os
+import unicodedata
 import wave
 from collections.abc import Iterable
 from pathlib import Path
@@ -11,13 +13,14 @@ from typing import Any, Dict, List, Optional, Tuple, Union, cast
 
 import numpy as np
 import torch
-import torchaudio
 from piper import PiperVoice, SynthesisConfig
 from piper.phonemize_espeak import EspeakPhonemizer
 
-from piper_train.vits import commons
+try:
+    from piper_train.vits import commons
+except ImportError:
+    from piper_train.vits import commons
 
-_DIR = Path(__file__).parent
 _LOGGER = logging.getLogger(__name__)
 logging.basicConfig(level=logging.DEBUG)
 
@@ -36,6 +39,7 @@ def generate_samples(
     noise_scale_ws: Tuple[float, ...] = (0.8,),
     max_speakers: Optional[int] = None,
     verbose: bool = False,
+    phoneme_input: bool = False,
     **kwargs,
 ) -> None:
     """
@@ -56,6 +60,7 @@ def generate_samples(
         noise_scale_ws (List[float]): A parameter for the stochastic duration of words/phonemes.
         max_speakers (int): The maximum speaker number to use, if the model is multi-speaker.
         verbose (bool): Enable or disable more detailed logging messages (default: False).
+        phoneme_input (bool): Set to indicate given input text is phoneme input.
     Returns:
         None
     """
@@ -73,6 +78,10 @@ def generate_samples(
     if torch.cuda.is_available():
         torch_model.cuda()
         _LOGGER.debug("CUDA available, using GPU")
+    elif torch.backends.mps.is_available():
+        mps_device = torch.device("mps")
+        torch_model.to(mps_device)
+        _LOGGER.debug("MPS available, using GPU")
 
     output_dir = Path(output_dir)
     output_dir.mkdir(parents=True, exist_ok=True)
@@ -102,7 +111,7 @@ def generate_samples(
 
     speakers_iter = it.cycle(it.product(range(num_speakers), range(num_speakers)))
     speakers_batch = list(it.islice(speakers_iter, 0, batch_size))
-    if isinstance(text, str) and os.path.exists(text):
+    if isinstance(text, str) and os.path.isfile(text):
         texts = it.cycle(
             [
                 i.strip()
@@ -132,7 +141,9 @@ def generate_samples(
 
             phoneme_ids_by_batch = []
             for i in range(batch_size):
-                phoneme_ids = get_phonemes(voice, config, next(texts), verbose)
+                phoneme_ids = get_phonemes(
+                    voice, config, next(texts), verbose, phoneme_input
+                )
                 phoneme_ids_by_batch.append(phoneme_ids)
 
             def right_pad_lists(lists):
@@ -146,23 +157,34 @@ def generate_samples(
                 return padded_lists
 
             phoneme_ids_by_batch = right_pad_lists(phoneme_ids_by_batch)
-            audio = (
-                generate_audio(
-                    torch_model,
-                    speaker_1,
-                    speaker_2,
-                    phoneme_ids_by_batch,
-                    slerp_weight,
-                    noise_scale,
-                    noise_scale_w,
-                    length_scale,
-                    max_len,
-                )
-                .cpu()
-                .numpy()
+            audio, phoneme_samples = generate_audio(
+                torch_model,
+                speaker_1,
+                speaker_2,
+                phoneme_ids_by_batch,
+                slerp_weight,
+                noise_scale,
+                noise_scale_w,
+                length_scale,
+                max_len,
             )
 
-            audio_int16 = audio_float_to_int16(audio)
+            # Trim audio to actual length based on phoneme samples
+            for i in range(audio.shape[0]):
+                # Fill time after last speech with silence (zeros)
+                # It will be removed in the next stage with np.trim_zeros
+                last_sample_idx = int(phoneme_samples[i].flatten().sum().item())
+                audio[i, 0, last_sample_idx + 1 :] = 0
+
+            audio_numpy = audio.cpu().numpy()
+
+            if torch.backends.mps.is_available():
+                # There seems to be a memory leak if we don't empty the cache
+                # after each batch with mps
+                torch.mps.empty_cache()
+                gc.collect()
+
+            audio_int16 = audio_float_to_int16(audio_numpy)
             for audio_idx in range(audio_int16.shape[0]):
                 audio_data = np.trim_zeros(audio_int16[audio_idx].flatten())
 
@@ -206,6 +228,7 @@ def generate_samples_onnx(
     noise_scales: Tuple[float, ...] = (0.667,),
     noise_scale_ws: Tuple[float, ...] = (0.8,),
     max_speakers: Optional[int] = None,
+    phoneme_input: bool = False,
     **kwargs,
 ) -> None:
     """
@@ -223,6 +246,7 @@ def generate_samples_onnx(
         noise_scales (List[float]): A parameter for overall variability of the generated speech.
         noise_scale_ws (List[float]): A parameter for the stochastic duration of words/phonemes.
         max_speakers (int): The maximum speaker number to use, if the model is multi-speaker.
+        phoneme_input (bool): Set to indicate given input text is phoneme input.
 
     Returns:
         None
@@ -278,17 +302,62 @@ def generate_samples_onnx(
             else:
                 wav_path = output_dir / f"{sample_idx}.wav"
 
-            wav_file: wave.Wave_write = wave.open(str(wav_path), "wb")
-            voice.synthesize_wav(
-                next(texts),
-                wav_file=wav_file,
-                syn_config=SynthesisConfig(
+            text_input = next(texts)
+
+            if phoneme_input:
+                # For ONNX models with phoneme input, build phoneme IDs manually
+                phonemes = list(unicodedata.normalize("NFD", text_input))
+
+                # Build phoneme IDs similar to get_phonemes function
+                id_map = voice.config.phoneme_id_map
+
+                # Beginning of utterance
+                phoneme_ids = list(id_map.get("^", [1]))  # Default to [1] if not found
+                phoneme_ids.extend(id_map.get("_", [0]))  # Default to [0] if not found
+
+                # Add phonemes
+                for phoneme in phonemes:
+                    p_ids = id_map.get(phoneme)
+                    if p_ids is not None:
+                        phoneme_ids.extend(p_ids)
+                        phoneme_ids.extend(id_map.get("_", [0]))
+                    else:
+                        _LOGGER.warning(
+                            "Phoneme '%s' not found in model's phoneme map", phoneme
+                        )
+
+                # End of utterance
+                phoneme_ids.extend(id_map.get("$", [2]))  # Default to [2] if not found
+
+                # Generate audio from phoneme IDs
+                syn_config = SynthesisConfig(
                     speaker_id=speaker_id,
                     length_scale=length_scale,
                     noise_scale=noise_scale,
                     noise_w_scale=noise_w_scale,
-                ),
-            )
+                )
+                audio = voice.phoneme_ids_to_audio(phoneme_ids, syn_config)
+
+                # Convert to int16 and write to WAV
+                audio_int16 = audio_float_to_int16(audio[np.newaxis, :])
+                wav_file: wave.Wave_write = wave.open(str(wav_path), "wb")
+                with wav_file:
+                    wav_file.setframerate(voice.config.sample_rate)
+                    wav_file.setsampwidth(2)
+                    wav_file.setnchannels(1)
+                    wav_file.writeframes(audio_int16.flatten())
+            else:
+                with wave.open(str(wav_path), "wb") as wav_file:
+                    voice.synthesize_wav(
+                        text_input,
+                        wav_file=wav_file,
+                        syn_config=SynthesisConfig(
+                            speaker_id=speaker_id,
+                            length_scale=length_scale,
+                            noise_scale=noise_scale,
+                            noise_w_scale=noise_w_scale,
+                        ),
+                    )
 
             sample_idx += 1
             if sample_idx >= max_samples:
@@ -310,7 +379,7 @@ def generate_audio(
     noise_scale_w,
     length_scale,
     max_len,
-) -> torch.FloatTensor:
+) -> Tuple[torch.FloatTensor, torch.FloatTensor]:
     x = torch.LongTensor(phoneme_ids)
     x_lengths = torch.LongTensor([len(i) for i in phoneme_ids])
 
@@ -319,6 +388,12 @@ def generate_audio(
         speaker_2 = speaker_2.cuda()
         x = cast(torch.LongTensor, x.cuda())
         x_lengths = cast(torch.LongTensor, x_lengths.cuda())
+    elif torch.backends.mps.is_available():
+        mps_device = torch.device("mps")
+        speaker_1 = speaker_1.to(mps_device)
+        speaker_2 = speaker_2.to(mps_device)
+        x = cast(torch.LongTensor, x.to(mps_device))
+        x_lengths = cast(torch.LongTensor, x_lengths.to(mps_device))
 
     x, m_p_orig, logs_p_orig, x_mask = model.enc_p(x, x_lengths)
     emb0 = model.emb_g(speaker_1)
@@ -349,9 +424,10 @@ def generate_audio(
     z = model.flow(z_p, y_mask, g=g, reverse=True)
     o = model.dec((z * y_mask)[:, :, :max_len], g=g)
 
-    audio = o
+    audio = cast(torch.FloatTensor, o)
+    phoneme_samples = cast(torch.FloatTensor, w_ceil * 256)  # hop length
 
-    return audio
+    return audio, phoneme_samples
 
 
 _PHONEMIZER = EspeakPhonemizer()
@@ -362,13 +438,17 @@ def get_phonemes(
     config: Dict[str, Any],
     text: str,
     verbose: bool = False,
+    phoneme_input: bool = False,
 ) -> List[int]:
     # Combine all sentences
-    phonemes = [
-        p
-        for sentence_phonemes in _PHONEMIZER.phonemize(voice, text)
-        for p in sentence_phonemes
-    ]
+    if phoneme_input:
+        phonemes = list(unicodedata.normalize("NFD", text))
+    else:
+        phonemes = [
+            p
+            for sentence_phonemes in _PHONEMIZER.phonemize(voice, text)
+            for p in sentence_phonemes
+        ]
     if verbose is True:
         _LOGGER.debug("Phonemes: %s", phonemes)
 
@@ -512,6 +592,9 @@ def main() -> int:
         type=int,
         help="Maximum number of speakers to use (default: no limit)",
     )
+    parser.add_argument(
+        "--phoneme-input", action="store_true", help="Treat input text as phoneme input"
+    )
     parser.add_argument("--verbose", action="store_true")
     args = parser.parse_args().__dict__
 

piper_sample_generator/augment.py

@@ -1,12 +1,11 @@
 #!/usr/bin/env python3
 import argparse
 import audioop
-import sys
 import wave
 from pathlib import Path
 
 import numpy as np
-from audiomentations import Compose, ApplyImpulseResponse, Gain
+from audiomentations import ApplyImpulseResponse, Compose, Gain
 
 _DIR = Path(__file__).parent
 
@@ -15,7 +14,7 @@ def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument("input_dir")
     parser.add_argument("output_dir")
-    parser.add_argument("--sample-rate", type=int)
+    parser.add_argument("--sample-rate", type=int, required=True)
     args = parser.parse_args()
 
     impulses = list((_DIR / "impulses").glob("*.wav"))
@@ -35,9 +34,10 @@ def main() -> None:
         output_wav = output_dir / (input_wav.relative_to(input_dir))
         output_wav.parent.mkdir(parents=True, exist_ok=True)
 
-        with wave.open(str(input_wav), "rb") as input_wav_file, wave.open(
-            str(output_wav), "wb"
-        ) as output_wav_file:
+        with (
+            wave.open(str(input_wav), "rb") as input_wav_file,
+            wave.open(str(output_wav), "wb") as output_wav_file,
+        ):
             assert input_wav_file.getsampwidth() == 2
             assert input_wav_file.getnchannels() == 1
 

pylintrc

@@ -1,3 +1,6 @@
+[MASTER]
+ignored-modules=torch
+
 [MESSAGES CONTROL]
 disable=
   format,
@@ -31,14 +34,7 @@ disable=
   missing-class-docstring,
   missing-function-docstring,
   import-error,
-  consider-using-with
+  relative-beyond-top-level
 
 [FORMAT]
 expected-line-ending-format=LF
-
-[TYPECHECK]
-
-# List of members which are set dynamically and missed by pylint inference
-# system, and so shouldn't trigger E1101 when accessed. Python regular
-# expressions are accepted.
-generated-members=numpy.*,torch.*

pyproject.toml

@@ -4,41 +4,31 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name        = "piper-sample-generator"
-version     = "3.0.0"
-license     = {text = "Apache-2.0"}
+version     = "3.2.0"
+license     = {text = "MIT"}
 description = "Generate TTS audio samples for training wake word systems"
 readme      = "README.md"
 authors     = [
     {name = "The Home Assistant Authors", email = "hello@home-assistant.io"}
 ]
 keywords    = ["piper", "sample", "tts", "wakeword"]
-classifiers = [
-    "Development Status :: 3 - Alpha",
-    "Intended Audience :: Developers",
-    "Topic :: Text Processing :: Linguistic",
-    "License :: OSI Approved :: Apache Software License",
-    "Programming Language :: Python :: 3.9",
-    "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.12",
-    "Programming Language :: Python :: 3.13",
-]
 requires-python = ">=3.9.0"
 dependencies = [
-    "piper-tts>=1.3.0,<2",
+    "audiomentations==0.33.0",
+    "piper-tts==1.3.0",
+    "numpy>=2,<3",
     "torch>=2,<3",
     "torchaudio",
-    "audiomentations",
-    "numpy",
+    "webrtcvad",
 ]
 
 [project.optional-dependencies]
 dev = [
-    "black==24.8.0",
-    "flake8==7.2.0",
-    "mypy==1.14.0",
-    "pylint==3.2.7",
-    "pytest==8.3.5",
+    "black==22.12.0",
+    "flake8==6.0.0",
+    "isort==5.11.3",
+    "mypy==0.991",
+    "pylint==2.15.9",
 ]
 
 [project.urls]
@@ -49,4 +39,7 @@ platforms = ["any"]
 zip-safe  = true
 
 [tool.setuptools.packages.find]
-include = []
+include = ["piper_sample_generator*"]
+
+[tool.setuptools.package-data]
+piper_sample_generator = ["impulses/*.wav"]

requirements.txt

@@ -1,6 +0,0 @@
-audiomentations==0.33.0
-piper-phonemize==1.1.0
-numpy<2
-torch<2
-torchaudio
-webrtcvad

requirements_dev.txt

@@ -1,5 +0,0 @@
-black==22.12.0
-flake8==6.0.0
-isort==5.11.3
-mypy==0.991
-pylint==2.15.9

script/format

@@ -6,7 +6,7 @@ from pathlib import Path
 _DIR = Path(__file__).parent
 _PROGRAM_DIR = _DIR.parent
 _VENV_DIR = _PROGRAM_DIR / ".venv"
-_SCRIPT = _PROGRAM_DIR / "generate_samples.py"
+_MODULE_DIR = _PROGRAM_DIR / "piper_sample_generator"
 
 if _VENV_DIR.exists():
     context = venv.EnvBuilder().ensure_directories(_VENV_DIR)
@@ -14,5 +14,5 @@ if _VENV_DIR.exists():
 else:
     python_exe = "python3"
 
-subprocess.check_call([python_exe, "-m", "black", str(_SCRIPT)])
-subprocess.check_call([python_exe, "-m", "isort", str(_SCRIPT)])
+subprocess.check_call([python_exe, "-m", "black", str(_MODULE_DIR)])
+subprocess.check_call([python_exe, "-m", "isort", str(_MODULE_DIR)])

script/lint

@@ -6,7 +6,7 @@ from pathlib import Path
 _DIR = Path(__file__).parent
 _PROGRAM_DIR = _DIR.parent
 _VENV_DIR = _PROGRAM_DIR / ".venv"
-_SCRIPT = _PROGRAM_DIR / "generate_samples.py"
+_MODULE_DIR = _PROGRAM_DIR / "piper_sample_generator"
 
 if _VENV_DIR.exists():
     context = venv.EnvBuilder().ensure_directories(_VENV_DIR)
@@ -14,8 +14,8 @@ if _VENV_DIR.exists():
 else:
     python_exe = "python3"
 
-subprocess.check_call([python_exe, "-m", "black", str(_SCRIPT), "--check"])
-subprocess.check_call([python_exe, "-m", "isort", str(_SCRIPT), "--check"])
-subprocess.check_call([python_exe, "-m", "flake8", str(_SCRIPT)])
-subprocess.check_call([python_exe, "-m", "pylint", str(_SCRIPT)])
-subprocess.check_call([python_exe, "-m", "mypy", str(_SCRIPT)])
+subprocess.check_call([python_exe, "-m", "black", str(_MODULE_DIR), "--check"])
+subprocess.check_call([python_exe, "-m", "isort", str(_MODULE_DIR), "--check"])
+subprocess.check_call([python_exe, "-m", "flake8", str(_MODULE_DIR)])
+subprocess.check_call([python_exe, "-m", "pylint", str(_MODULE_DIR)])
+subprocess.check_call([python_exe, "-m", "mypy", str(_MODULE_DIR)])

script/run

@@ -14,4 +14,4 @@ if _VENV_DIR.exists():
 else:
     python_exe = "python3"
 
-subprocess.check_call([python_exe, "generate_samples.py"] + sys.argv[1:])
+subprocess.check_call([python_exe, "-m", "piper_sample_generator"] + sys.argv[1:])