diff --git a/CHANGELOG.md b/CHANGELOG.md index d3bb09a..916341d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,19 @@ # Changelog +## 3.2.0 + +- Refactor as `piper_sample_generator` package + +## 3.1.0 + +- Support MPS acceleration on Apple Silicon +- Add `--phoneme-input` flag + ## 3.0.0 - Move phonemization to piper 1.3.0 (piper-phonemize is deprecated) - Move to PyTorch 2 - Add support for using Piper voices (`.onnx`) directly +- Allow multiple `--model` for Piper voices (`.onnx`) - Remove silence trimming - Remove `min-phoneme-count` diff --git a/README.md b/README.md index fdc4117..944d0cf 100644 --- a/README.md +++ b/README.md @@ -6,16 +6,8 @@ Supports normal [Piper voices][piper voices] or a special [generator][] that can ## Install -Create a virtual environment and install the requirements: - ``` sh -git clone https://github.com/rhasspy/piper-sample-generator.git -cd piper-sample-generator/ - -python3 -m venv .venv -source .venv/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip install -e . +pip install piper-sample-generator ``` ## Piper Voices @@ -33,7 +25,7 @@ wget -O voices/en_US-lessac-medium.onnx.json 'https://huggingface.co/rhasspy/pip Generate a small set of samples with the CLI: ``` sh -python3 generate_samples.py 'okay piper.' --model voices/en_US-lessac-medium.onnx --max-samples 10 --output-dir okay_piper/ +python3 -m piper_sample_generator 'okay piper.' --model voices/en_US-lessac-medium.onnx --max-samples 10 --output-dir okay_piper/ ``` Check the `okay_piper/` directory for 10 WAV files (named `0.wav` to `9.wav`). @@ -53,7 +45,7 @@ wget -O models/en-us-libritts-high.pt 'https://github.com/rhasspy/piper-sample-g Generate a small set of samples with the CLI: ``` sh -python3 generate_samples.py 'okay piper.' --model models/en-us-libritts-high.pt --max-samples 10 --output-dir okay_piper/ +python3 -m piper_sample_generator 'okay piper.' --model models/en-us-libritts-high.pt --max-samples 10 --output-dir okay_piper/ ``` Check the `okay_piper/` directory for 10 WAV files (named `0.wav` to `9.wav`). @@ -61,7 +53,7 @@ Check the `okay_piper/` directory for 10 WAV files (named `0.wav` to `9.wav`). Generation can be much faster and more efficient if you have a GPU available and PyTorch is configured to use it. In this case, increase the batch size: ``` sh -python3 generate_samples.py 'okay piper.' --model models/en-us-libritts-high.pt --max-samples 100 --batch-size 10 --output-dir okay_piper/ +python3 -m piper_sample_generator 'okay piper.' --model models/en-us-libritts-high.pt --max-samples 100 --batch-size 10 --output-dir okay_piper/ ``` On an NVidia 2080 Ti with 11GB, a batch size of 100 was possible (generating approximately 100 samples per second). @@ -75,14 +67,14 @@ See `--help` for more options, including the `--length-scales` (speaking speeds) Once you have samples generated, you can augment them using [audiomentation](https://iver56.github.io/audiomentations/): ``` sh -python3 augment.py --sample-rate 22050 okay_piper/ okay_piper_augmented/ +python3 -m piper_sample_generator.augment --sample-rate 22050 okay_piper/ okay_piper_augmented/ ``` This will do several things to each sample: 1. Randomly decrease the volume * The original samples are normalized, so different volume levels are needed -2. Randomly apply an [impulse response][] using the files in `impulses/` +2. Randomly apply an [impulse response][] using the files in `piper_sample_generator/impulses/` * Change the acoustics of the sample to sound like the speaker was in a room with echo or using a poor quality microphone 3. Resample to 16Khz for training (e.g., [openWakeWord][]) diff --git a/piper_sample_generator/__init__.py b/piper_sample_generator/__init__.py new file mode 100644 index 0000000..0142775 --- /dev/null +++ b/piper_sample_generator/__init__.py @@ -0,0 +1 @@ +"""Piper sample generator.""" diff --git a/generate_samples.py b/piper_sample_generator/__main__.py similarity index 77% rename from generate_samples.py rename to piper_sample_generator/__main__.py index 57c09c2..3074fb5 100755 --- a/generate_samples.py +++ b/piper_sample_generator/__main__.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 import argparse +import gc import itertools as it import json import logging import os +import unicodedata import wave from collections.abc import Iterable from pathlib import Path @@ -11,13 +13,14 @@ from typing import Any, Dict, List, Optional, Tuple, Union, cast import numpy as np import torch -import torchaudio from piper import PiperVoice, SynthesisConfig from piper.phonemize_espeak import EspeakPhonemizer -from piper_train.vits import commons +try: + from piper_train.vits import commons +except ImportError: + from piper_train.vits import commons -_DIR = Path(__file__).parent _LOGGER = logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG) @@ -36,6 +39,7 @@ def generate_samples( noise_scale_ws: Tuple[float, ...] = (0.8,), max_speakers: Optional[int] = None, verbose: bool = False, + phoneme_input: bool = False, **kwargs, ) -> None: """ @@ -56,6 +60,7 @@ def generate_samples( noise_scale_ws (List[float]): A parameter for the stochastic duration of words/phonemes. max_speakers (int): The maximum speaker number to use, if the model is multi-speaker. verbose (bool): Enable or disable more detailed logging messages (default: False). + phoneme_input (bool): Set to indicate given input text is phoneme input. Returns: None """ @@ -73,6 +78,10 @@ def generate_samples( if torch.cuda.is_available(): torch_model.cuda() _LOGGER.debug("CUDA available, using GPU") + elif torch.backends.mps.is_available(): + mps_device = torch.device("mps") + torch_model.to(mps_device) + _LOGGER.debug("MPS available, using GPU") output_dir = Path(output_dir) output_dir.mkdir(parents=True, exist_ok=True) @@ -102,7 +111,7 @@ def generate_samples( speakers_iter = it.cycle(it.product(range(num_speakers), range(num_speakers))) speakers_batch = list(it.islice(speakers_iter, 0, batch_size)) - if isinstance(text, str) and os.path.exists(text): + if isinstance(text, str) and os.path.isfile(text): texts = it.cycle( [ i.strip() @@ -132,7 +141,9 @@ def generate_samples( phoneme_ids_by_batch = [] for i in range(batch_size): - phoneme_ids = get_phonemes(voice, config, next(texts), verbose) + phoneme_ids = get_phonemes( + voice, config, next(texts), verbose, phoneme_input + ) phoneme_ids_by_batch.append(phoneme_ids) def right_pad_lists(lists): @@ -146,23 +157,34 @@ def generate_samples( return padded_lists phoneme_ids_by_batch = right_pad_lists(phoneme_ids_by_batch) - audio = ( - generate_audio( - torch_model, - speaker_1, - speaker_2, - phoneme_ids_by_batch, - slerp_weight, - noise_scale, - noise_scale_w, - length_scale, - max_len, - ) - .cpu() - .numpy() + audio, phoneme_samples = generate_audio( + torch_model, + speaker_1, + speaker_2, + phoneme_ids_by_batch, + slerp_weight, + noise_scale, + noise_scale_w, + length_scale, + max_len, ) - audio_int16 = audio_float_to_int16(audio) + # Trim audio to actual length based on phoneme samples + for i in range(audio.shape[0]): + # Fill time after last speech with silence (zeros) + # It will be removed in the next stage with np.trim_zeros + last_sample_idx = int(phoneme_samples[i].flatten().sum().item()) + audio[i, 0, last_sample_idx + 1 :] = 0 + + audio_numpy = audio.cpu().numpy() + + if torch.backends.mps.is_available(): + # There seems to be a memory leak if we don't empty the cache + # after each batch with mps + torch.mps.empty_cache() + gc.collect() + + audio_int16 = audio_float_to_int16(audio_numpy) for audio_idx in range(audio_int16.shape[0]): audio_data = np.trim_zeros(audio_int16[audio_idx].flatten()) @@ -206,6 +228,7 @@ def generate_samples_onnx( noise_scales: Tuple[float, ...] = (0.667,), noise_scale_ws: Tuple[float, ...] = (0.8,), max_speakers: Optional[int] = None, + phoneme_input: bool = False, **kwargs, ) -> None: """ @@ -223,6 +246,7 @@ def generate_samples_onnx( noise_scales (List[float]): A parameter for overall variability of the generated speech. noise_scale_ws (List[float]): A parameter for the stochastic duration of words/phonemes. max_speakers (int): The maximum speaker number to use, if the model is multi-speaker. + phoneme_input (bool): Set to indicate given input text is phoneme input. Returns: None @@ -278,17 +302,62 @@ def generate_samples_onnx( else: wav_path = output_dir / f"{sample_idx}.wav" - wav_file: wave.Wave_write = wave.open(str(wav_path), "wb") - voice.synthesize_wav( - next(texts), - wav_file=wav_file, - syn_config=SynthesisConfig( + text_input = next(texts) + + if phoneme_input: + # For ONNX models with phoneme input, build phoneme IDs manually + phonemes = list(unicodedata.normalize("NFD", text_input)) + + # Build phoneme IDs similar to get_phonemes function + id_map = voice.config.phoneme_id_map + + # Beginning of utterance + phoneme_ids = list(id_map.get("^", [1])) # Default to [1] if not found + phoneme_ids.extend(id_map.get("_", [0])) # Default to [0] if not found + + # Add phonemes + for phoneme in phonemes: + p_ids = id_map.get(phoneme) + if p_ids is not None: + phoneme_ids.extend(p_ids) + phoneme_ids.extend(id_map.get("_", [0])) + else: + _LOGGER.warning( + "Phoneme '%s' not found in model's phoneme map", phoneme + ) + + # End of utterance + phoneme_ids.extend(id_map.get("$", [2])) # Default to [2] if not found + + # Generate audio from phoneme IDs + syn_config = SynthesisConfig( speaker_id=speaker_id, length_scale=length_scale, noise_scale=noise_scale, noise_w_scale=noise_w_scale, - ), - ) + ) + audio = voice.phoneme_ids_to_audio(phoneme_ids, syn_config) + + # Convert to int16 and write to WAV + audio_int16 = audio_float_to_int16(audio[np.newaxis, :]) + wav_file: wave.Wave_write = wave.open(str(wav_path), "wb") + with wav_file: + wav_file.setframerate(voice.config.sample_rate) + wav_file.setsampwidth(2) + wav_file.setnchannels(1) + wav_file.writeframes(audio_int16.flatten()) + else: + with wave.open(str(wav_path), "wb") as wav_file: + voice.synthesize_wav( + text_input, + wav_file=wav_file, + syn_config=SynthesisConfig( + speaker_id=speaker_id, + length_scale=length_scale, + noise_scale=noise_scale, + noise_w_scale=noise_w_scale, + ), + ) sample_idx += 1 if sample_idx >= max_samples: @@ -310,7 +379,7 @@ def generate_audio( noise_scale_w, length_scale, max_len, -) -> torch.FloatTensor: +) -> Tuple[torch.FloatTensor, torch.FloatTensor]: x = torch.LongTensor(phoneme_ids) x_lengths = torch.LongTensor([len(i) for i in phoneme_ids]) @@ -319,6 +388,12 @@ def generate_audio( speaker_2 = speaker_2.cuda() x = cast(torch.LongTensor, x.cuda()) x_lengths = cast(torch.LongTensor, x_lengths.cuda()) + elif torch.backends.mps.is_available(): + mps_device = torch.device("mps") + speaker_1 = speaker_1.to(mps_device) + speaker_2 = speaker_2.to(mps_device) + x = cast(torch.LongTensor, x.to(mps_device)) + x_lengths = cast(torch.LongTensor, x_lengths.to(mps_device)) x, m_p_orig, logs_p_orig, x_mask = model.enc_p(x, x_lengths) emb0 = model.emb_g(speaker_1) @@ -349,9 +424,10 @@ def generate_audio( z = model.flow(z_p, y_mask, g=g, reverse=True) o = model.dec((z * y_mask)[:, :, :max_len], g=g) - audio = o + audio = cast(torch.FloatTensor, o) + phoneme_samples = cast(torch.FloatTensor, w_ceil * 256) # hop length - return audio + return audio, phoneme_samples _PHONEMIZER = EspeakPhonemizer() @@ -362,13 +438,17 @@ def get_phonemes( config: Dict[str, Any], text: str, verbose: bool = False, + phoneme_input: bool = False, ) -> List[int]: # Combine all sentences - phonemes = [ - p - for sentence_phonemes in _PHONEMIZER.phonemize(voice, text) - for p in sentence_phonemes - ] + if phoneme_input: + phonemes = list(unicodedata.normalize("NFD", text)) + else: + phonemes = [ + p + for sentence_phonemes in _PHONEMIZER.phonemize(voice, text) + for p in sentence_phonemes + ] if verbose is True: _LOGGER.debug("Phonemes: %s", phonemes) @@ -512,6 +592,9 @@ def main() -> int: type=int, help="Maximum number of speakers to use (default: no limit)", ) + parser.add_argument( + "--phoneme-input", action="store_true", help="Treat input text as phoneme input" + ) parser.add_argument("--verbose", action="store_true") args = parser.parse_args().__dict__ diff --git a/augment.py b/piper_sample_generator/augment.py similarity index 89% rename from augment.py rename to piper_sample_generator/augment.py index 269b9e9..47a226d 100644 --- a/augment.py +++ b/piper_sample_generator/augment.py @@ -1,12 +1,11 @@ #!/usr/bin/env python3 import argparse import audioop -import sys import wave from pathlib import Path import numpy as np -from audiomentations import Compose, ApplyImpulseResponse, Gain +from audiomentations import ApplyImpulseResponse, Compose, Gain _DIR = Path(__file__).parent @@ -15,7 +14,7 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("input_dir") parser.add_argument("output_dir") - parser.add_argument("--sample-rate", type=int) + parser.add_argument("--sample-rate", type=int, required=True) args = parser.parse_args() impulses = list((_DIR / "impulses").glob("*.wav")) @@ -35,9 +34,10 @@ def main() -> None: output_wav = output_dir / (input_wav.relative_to(input_dir)) output_wav.parent.mkdir(parents=True, exist_ok=True) - with wave.open(str(input_wav), "rb") as input_wav_file, wave.open( - str(output_wav), "wb" - ) as output_wav_file: + with ( + wave.open(str(input_wav), "rb") as input_wav_file, + wave.open(str(output_wav), "wb") as output_wav_file, + ): assert input_wav_file.getsampwidth() == 2 assert input_wav_file.getnchannels() == 1 diff --git a/impulses/Accoustic2_Impulse.wav b/piper_sample_generator/impulses/Accoustic2_Impulse.wav similarity index 100% rename from impulses/Accoustic2_Impulse.wav rename to piper_sample_generator/impulses/Accoustic2_Impulse.wav diff --git a/impulses/Blatty Plate.wav b/piper_sample_generator/impulses/Blatty Plate.wav similarity index 100% rename from impulses/Blatty Plate.wav rename to piper_sample_generator/impulses/Blatty Plate.wav diff --git a/impulses/Concrete Room.wav b/piper_sample_generator/impulses/Concrete Room.wav similarity index 100% rename from impulses/Concrete Room.wav rename to piper_sample_generator/impulses/Concrete Room.wav diff --git a/impulses/Derlon Sanctuary.wav b/piper_sample_generator/impulses/Derlon Sanctuary.wav similarity index 100% rename from impulses/Derlon Sanctuary.wav rename to piper_sample_generator/impulses/Derlon Sanctuary.wav diff --git a/impulses/Fat Bass.wav b/piper_sample_generator/impulses/Fat Bass.wav similarity index 100% rename from impulses/Fat Bass.wav rename to piper_sample_generator/impulses/Fat Bass.wav diff --git a/impulses/Reverse Gate.wav b/piper_sample_generator/impulses/Reverse Gate.wav similarity index 100% rename from impulses/Reverse Gate.wav rename to piper_sample_generator/impulses/Reverse Gate.wav diff --git a/impulses/Symphonic.wav b/piper_sample_generator/impulses/Symphonic.wav similarity index 100% rename from impulses/Symphonic.wav rename to piper_sample_generator/impulses/Symphonic.wav diff --git a/impulses/ir_bathroom1.wav b/piper_sample_generator/impulses/ir_bathroom1.wav similarity index 100% rename from impulses/ir_bathroom1.wav rename to piper_sample_generator/impulses/ir_bathroom1.wav diff --git a/pylintrc b/pylintrc index 561b2f1..60fdb1d 100644 --- a/pylintrc +++ b/pylintrc @@ -1,3 +1,6 @@ +[MASTER] +ignored-modules=torch + [MESSAGES CONTROL] disable= format, @@ -31,14 +34,7 @@ disable= missing-class-docstring, missing-function-docstring, import-error, - consider-using-with + relative-beyond-top-level [FORMAT] expected-line-ending-format=LF - -[TYPECHECK] - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members=numpy.*,torch.* diff --git a/pyproject.toml b/pyproject.toml index 5061a05..76cf7d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,41 +4,31 @@ build-backend = "setuptools.build_meta" [project] name = "piper-sample-generator" -version = "3.0.0" -license = {text = "Apache-2.0"} +version = "3.2.0" +license = {text = "MIT"} description = "Generate TTS audio samples for training wake word systems" readme = "README.md" authors = [ {name = "The Home Assistant Authors", email = "hello@home-assistant.io"} ] keywords = ["piper", "sample", "tts", "wakeword"] -classifiers = [ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "Topic :: Text Processing :: Linguistic", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", -] requires-python = ">=3.9.0" dependencies = [ - "piper-tts>=1.3.0,<2", + "audiomentations==0.33.0", + "piper-tts==1.3.0", + "numpy>=2,<3", "torch>=2,<3", "torchaudio", - "audiomentations", - "numpy", + "webrtcvad", ] [project.optional-dependencies] dev = [ - "black==24.8.0", - "flake8==7.2.0", - "mypy==1.14.0", - "pylint==3.2.7", - "pytest==8.3.5", + "black==22.12.0", + "flake8==6.0.0", + "isort==5.11.3", + "mypy==0.991", + "pylint==2.15.9", ] [project.urls] @@ -49,4 +39,7 @@ platforms = ["any"] zip-safe = true [tool.setuptools.packages.find] -include = [] +include = ["piper_sample_generator*"] + +[tool.setuptools.package-data] +piper_sample_generator = ["impulses/*.wav"] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 8443d84..0000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -audiomentations==0.33.0 -piper-phonemize==1.1.0 -numpy<2 -torch<2 -torchaudio -webrtcvad diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index 77190e6..0000000 --- a/requirements_dev.txt +++ /dev/null @@ -1,5 +0,0 @@ -black==22.12.0 -flake8==6.0.0 -isort==5.11.3 -mypy==0.991 -pylint==2.15.9 diff --git a/script/format b/script/format index 7f04417..b8b283b 100755 --- a/script/format +++ b/script/format @@ -6,7 +6,7 @@ from pathlib import Path _DIR = Path(__file__).parent _PROGRAM_DIR = _DIR.parent _VENV_DIR = _PROGRAM_DIR / ".venv" -_SCRIPT = _PROGRAM_DIR / "generate_samples.py" +_MODULE_DIR = _PROGRAM_DIR / "piper_sample_generator" if _VENV_DIR.exists(): context = venv.EnvBuilder().ensure_directories(_VENV_DIR) @@ -14,5 +14,5 @@ if _VENV_DIR.exists(): else: python_exe = "python3" -subprocess.check_call([python_exe, "-m", "black", str(_SCRIPT)]) -subprocess.check_call([python_exe, "-m", "isort", str(_SCRIPT)]) +subprocess.check_call([python_exe, "-m", "black", str(_MODULE_DIR)]) +subprocess.check_call([python_exe, "-m", "isort", str(_MODULE_DIR)]) diff --git a/script/lint b/script/lint index e4231e0..34222f0 100755 --- a/script/lint +++ b/script/lint @@ -6,7 +6,7 @@ from pathlib import Path _DIR = Path(__file__).parent _PROGRAM_DIR = _DIR.parent _VENV_DIR = _PROGRAM_DIR / ".venv" -_SCRIPT = _PROGRAM_DIR / "generate_samples.py" +_MODULE_DIR = _PROGRAM_DIR / "piper_sample_generator" if _VENV_DIR.exists(): context = venv.EnvBuilder().ensure_directories(_VENV_DIR) @@ -14,8 +14,8 @@ if _VENV_DIR.exists(): else: python_exe = "python3" -subprocess.check_call([python_exe, "-m", "black", str(_SCRIPT), "--check"]) -subprocess.check_call([python_exe, "-m", "isort", str(_SCRIPT), "--check"]) -subprocess.check_call([python_exe, "-m", "flake8", str(_SCRIPT)]) -subprocess.check_call([python_exe, "-m", "pylint", str(_SCRIPT)]) -subprocess.check_call([python_exe, "-m", "mypy", str(_SCRIPT)]) +subprocess.check_call([python_exe, "-m", "black", str(_MODULE_DIR), "--check"]) +subprocess.check_call([python_exe, "-m", "isort", str(_MODULE_DIR), "--check"]) +subprocess.check_call([python_exe, "-m", "flake8", str(_MODULE_DIR)]) +subprocess.check_call([python_exe, "-m", "pylint", str(_MODULE_DIR)]) +subprocess.check_call([python_exe, "-m", "mypy", str(_MODULE_DIR)]) diff --git a/script/run b/script/run index f2837d0..ae921c9 100755 --- a/script/run +++ b/script/run @@ -14,4 +14,4 @@ if _VENV_DIR.exists(): else: python_exe = "python3" -subprocess.check_call([python_exe, "generate_samples.py"] + sys.argv[1:]) +subprocess.check_call([python_exe, "-m", "piper_sample_generator"] + sys.argv[1:])