diff --git a/CHANGELOG.md b/CHANGELOG.md index 916341d..d3bb09a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,19 +1,9 @@ # Changelog -## 3.2.0 - -- Refactor as `piper_sample_generator` package - -## 3.1.0 - -- Support MPS acceleration on Apple Silicon -- Add `--phoneme-input` flag - ## 3.0.0 - Move phonemization to piper 1.3.0 (piper-phonemize is deprecated) - Move to PyTorch 2 - Add support for using Piper voices (`.onnx`) directly -- Allow multiple `--model` for Piper voices (`.onnx`) - Remove silence trimming - Remove `min-phoneme-count` diff --git a/README.md b/README.md index 944d0cf..fdc4117 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,16 @@ Supports normal [Piper voices][piper voices] or a special [generator][] that can ## Install +Create a virtual environment and install the requirements: + ``` sh -pip install piper-sample-generator +git clone https://github.com/rhasspy/piper-sample-generator.git +cd piper-sample-generator/ + +python3 -m venv .venv +source .venv/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip install -e . ``` ## Piper Voices @@ -25,7 +33,7 @@ wget -O voices/en_US-lessac-medium.onnx.json 'https://huggingface.co/rhasspy/pip Generate a small set of samples with the CLI: ``` sh -python3 -m piper_sample_generator 'okay piper.' --model voices/en_US-lessac-medium.onnx --max-samples 10 --output-dir okay_piper/ +python3 generate_samples.py 'okay piper.' --model voices/en_US-lessac-medium.onnx --max-samples 10 --output-dir okay_piper/ ``` Check the `okay_piper/` directory for 10 WAV files (named `0.wav` to `9.wav`). @@ -45,7 +53,7 @@ wget -O models/en-us-libritts-high.pt 'https://github.com/rhasspy/piper-sample-g Generate a small set of samples with the CLI: ``` sh -python3 -m piper_sample_generator 'okay piper.' --model models/en-us-libritts-high.pt --max-samples 10 --output-dir okay_piper/ +python3 generate_samples.py 'okay piper.' --model models/en-us-libritts-high.pt --max-samples 10 --output-dir okay_piper/ ``` Check the `okay_piper/` directory for 10 WAV files (named `0.wav` to `9.wav`). @@ -53,7 +61,7 @@ Check the `okay_piper/` directory for 10 WAV files (named `0.wav` to `9.wav`). Generation can be much faster and more efficient if you have a GPU available and PyTorch is configured to use it. In this case, increase the batch size: ``` sh -python3 -m piper_sample_generator 'okay piper.' --model models/en-us-libritts-high.pt --max-samples 100 --batch-size 10 --output-dir okay_piper/ +python3 generate_samples.py 'okay piper.' --model models/en-us-libritts-high.pt --max-samples 100 --batch-size 10 --output-dir okay_piper/ ``` On an NVidia 2080 Ti with 11GB, a batch size of 100 was possible (generating approximately 100 samples per second). @@ -67,14 +75,14 @@ See `--help` for more options, including the `--length-scales` (speaking speeds) Once you have samples generated, you can augment them using [audiomentation](https://iver56.github.io/audiomentations/): ``` sh -python3 -m piper_sample_generator.augment --sample-rate 22050 okay_piper/ okay_piper_augmented/ +python3 augment.py --sample-rate 22050 okay_piper/ okay_piper_augmented/ ``` This will do several things to each sample: 1. Randomly decrease the volume * The original samples are normalized, so different volume levels are needed -2. Randomly apply an [impulse response][] using the files in `piper_sample_generator/impulses/` +2. Randomly apply an [impulse response][] using the files in `impulses/` * Change the acoustics of the sample to sound like the speaker was in a room with echo or using a poor quality microphone 3. Resample to 16Khz for training (e.g., [openWakeWord][]) diff --git a/piper_sample_generator/augment.py b/augment.py similarity index 89% rename from piper_sample_generator/augment.py rename to augment.py index 47a226d..269b9e9 100644 --- a/piper_sample_generator/augment.py +++ b/augment.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 import argparse import audioop +import sys import wave from pathlib import Path import numpy as np -from audiomentations import ApplyImpulseResponse, Compose, Gain +from audiomentations import Compose, ApplyImpulseResponse, Gain _DIR = Path(__file__).parent @@ -14,7 +15,7 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("input_dir") parser.add_argument("output_dir") - parser.add_argument("--sample-rate", type=int, required=True) + parser.add_argument("--sample-rate", type=int) args = parser.parse_args() impulses = list((_DIR / "impulses").glob("*.wav")) @@ -34,10 +35,9 @@ def main() -> None: output_wav = output_dir / (input_wav.relative_to(input_dir)) output_wav.parent.mkdir(parents=True, exist_ok=True) - with ( - wave.open(str(input_wav), "rb") as input_wav_file, - wave.open(str(output_wav), "wb") as output_wav_file, - ): + with wave.open(str(input_wav), "rb") as input_wav_file, wave.open( + str(output_wav), "wb" + ) as output_wav_file: assert input_wav_file.getsampwidth() == 2 assert input_wav_file.getnchannels() == 1 diff --git a/piper_sample_generator/__main__.py b/generate_samples.py similarity index 77% rename from piper_sample_generator/__main__.py rename to generate_samples.py index 3074fb5..57c09c2 100755 --- a/piper_sample_generator/__main__.py +++ b/generate_samples.py @@ -1,11 +1,9 @@ #!/usr/bin/env python3 import argparse -import gc import itertools as it import json import logging import os -import unicodedata import wave from collections.abc import Iterable from pathlib import Path @@ -13,14 +11,13 @@ from typing import Any, Dict, List, Optional, Tuple, Union, cast import numpy as np import torch +import torchaudio from piper import PiperVoice, SynthesisConfig from piper.phonemize_espeak import EspeakPhonemizer -try: - from piper_train.vits import commons -except ImportError: - from piper_train.vits import commons +from piper_train.vits import commons +_DIR = Path(__file__).parent _LOGGER = logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG) @@ -39,7 +36,6 @@ def generate_samples( noise_scale_ws: Tuple[float, ...] = (0.8,), max_speakers: Optional[int] = None, verbose: bool = False, - phoneme_input: bool = False, **kwargs, ) -> None: """ @@ -60,7 +56,6 @@ def generate_samples( noise_scale_ws (List[float]): A parameter for the stochastic duration of words/phonemes. max_speakers (int): The maximum speaker number to use, if the model is multi-speaker. verbose (bool): Enable or disable more detailed logging messages (default: False). - phoneme_input (bool): Set to indicate given input text is phoneme input. Returns: None """ @@ -78,10 +73,6 @@ def generate_samples( if torch.cuda.is_available(): torch_model.cuda() _LOGGER.debug("CUDA available, using GPU") - elif torch.backends.mps.is_available(): - mps_device = torch.device("mps") - torch_model.to(mps_device) - _LOGGER.debug("MPS available, using GPU") output_dir = Path(output_dir) output_dir.mkdir(parents=True, exist_ok=True) @@ -111,7 +102,7 @@ def generate_samples( speakers_iter = it.cycle(it.product(range(num_speakers), range(num_speakers))) speakers_batch = list(it.islice(speakers_iter, 0, batch_size)) - if isinstance(text, str) and os.path.isfile(text): + if isinstance(text, str) and os.path.exists(text): texts = it.cycle( [ i.strip() @@ -141,9 +132,7 @@ def generate_samples( phoneme_ids_by_batch = [] for i in range(batch_size): - phoneme_ids = get_phonemes( - voice, config, next(texts), verbose, phoneme_input - ) + phoneme_ids = get_phonemes(voice, config, next(texts), verbose) phoneme_ids_by_batch.append(phoneme_ids) def right_pad_lists(lists): @@ -157,34 +146,23 @@ def generate_samples( return padded_lists phoneme_ids_by_batch = right_pad_lists(phoneme_ids_by_batch) - audio, phoneme_samples = generate_audio( - torch_model, - speaker_1, - speaker_2, - phoneme_ids_by_batch, - slerp_weight, - noise_scale, - noise_scale_w, - length_scale, - max_len, + audio = ( + generate_audio( + torch_model, + speaker_1, + speaker_2, + phoneme_ids_by_batch, + slerp_weight, + noise_scale, + noise_scale_w, + length_scale, + max_len, + ) + .cpu() + .numpy() ) - # Trim audio to actual length based on phoneme samples - for i in range(audio.shape[0]): - # Fill time after last speech with silence (zeros) - # It will be removed in the next stage with np.trim_zeros - last_sample_idx = int(phoneme_samples[i].flatten().sum().item()) - audio[i, 0, last_sample_idx + 1 :] = 0 - - audio_numpy = audio.cpu().numpy() - - if torch.backends.mps.is_available(): - # There seems to be a memory leak if we don't empty the cache - # after each batch with mps - torch.mps.empty_cache() - gc.collect() - - audio_int16 = audio_float_to_int16(audio_numpy) + audio_int16 = audio_float_to_int16(audio) for audio_idx in range(audio_int16.shape[0]): audio_data = np.trim_zeros(audio_int16[audio_idx].flatten()) @@ -228,7 +206,6 @@ def generate_samples_onnx( noise_scales: Tuple[float, ...] = (0.667,), noise_scale_ws: Tuple[float, ...] = (0.8,), max_speakers: Optional[int] = None, - phoneme_input: bool = False, **kwargs, ) -> None: """ @@ -246,7 +223,6 @@ def generate_samples_onnx( noise_scales (List[float]): A parameter for overall variability of the generated speech. noise_scale_ws (List[float]): A parameter for the stochastic duration of words/phonemes. max_speakers (int): The maximum speaker number to use, if the model is multi-speaker. - phoneme_input (bool): Set to indicate given input text is phoneme input. Returns: None @@ -302,62 +278,17 @@ def generate_samples_onnx( else: wav_path = output_dir / f"{sample_idx}.wav" - text_input = next(texts) - - if phoneme_input: - # For ONNX models with phoneme input, build phoneme IDs manually - phonemes = list(unicodedata.normalize("NFD", text_input)) - - # Build phoneme IDs similar to get_phonemes function - id_map = voice.config.phoneme_id_map - - # Beginning of utterance - phoneme_ids = list(id_map.get("^", [1])) # Default to [1] if not found - phoneme_ids.extend(id_map.get("_", [0])) # Default to [0] if not found - - # Add phonemes - for phoneme in phonemes: - p_ids = id_map.get(phoneme) - if p_ids is not None: - phoneme_ids.extend(p_ids) - phoneme_ids.extend(id_map.get("_", [0])) - else: - _LOGGER.warning( - "Phoneme '%s' not found in model's phoneme map", phoneme - ) - - # End of utterance - phoneme_ids.extend(id_map.get("$", [2])) # Default to [2] if not found - - # Generate audio from phoneme IDs - syn_config = SynthesisConfig( + wav_file: wave.Wave_write = wave.open(str(wav_path), "wb") + voice.synthesize_wav( + next(texts), + wav_file=wav_file, + syn_config=SynthesisConfig( speaker_id=speaker_id, length_scale=length_scale, noise_scale=noise_scale, noise_w_scale=noise_w_scale, - ) - audio = voice.phoneme_ids_to_audio(phoneme_ids, syn_config) - - # Convert to int16 and write to WAV - audio_int16 = audio_float_to_int16(audio[np.newaxis, :]) - wav_file: wave.Wave_write = wave.open(str(wav_path), "wb") - with wav_file: - wav_file.setframerate(voice.config.sample_rate) - wav_file.setsampwidth(2) - wav_file.setnchannels(1) - wav_file.writeframes(audio_int16.flatten()) - else: - with wave.open(str(wav_path), "wb") as wav_file: - voice.synthesize_wav( - text_input, - wav_file=wav_file, - syn_config=SynthesisConfig( - speaker_id=speaker_id, - length_scale=length_scale, - noise_scale=noise_scale, - noise_w_scale=noise_w_scale, - ), - ) + ), + ) sample_idx += 1 if sample_idx >= max_samples: @@ -379,7 +310,7 @@ def generate_audio( noise_scale_w, length_scale, max_len, -) -> Tuple[torch.FloatTensor, torch.FloatTensor]: +) -> torch.FloatTensor: x = torch.LongTensor(phoneme_ids) x_lengths = torch.LongTensor([len(i) for i in phoneme_ids]) @@ -388,12 +319,6 @@ def generate_audio( speaker_2 = speaker_2.cuda() x = cast(torch.LongTensor, x.cuda()) x_lengths = cast(torch.LongTensor, x_lengths.cuda()) - elif torch.backends.mps.is_available(): - mps_device = torch.device("mps") - speaker_1 = speaker_1.to(mps_device) - speaker_2 = speaker_2.to(mps_device) - x = cast(torch.LongTensor, x.to(mps_device)) - x_lengths = cast(torch.LongTensor, x_lengths.to(mps_device)) x, m_p_orig, logs_p_orig, x_mask = model.enc_p(x, x_lengths) emb0 = model.emb_g(speaker_1) @@ -424,10 +349,9 @@ def generate_audio( z = model.flow(z_p, y_mask, g=g, reverse=True) o = model.dec((z * y_mask)[:, :, :max_len], g=g) - audio = cast(torch.FloatTensor, o) - phoneme_samples = cast(torch.FloatTensor, w_ceil * 256) # hop length + audio = o - return audio, phoneme_samples + return audio _PHONEMIZER = EspeakPhonemizer() @@ -438,17 +362,13 @@ def get_phonemes( config: Dict[str, Any], text: str, verbose: bool = False, - phoneme_input: bool = False, ) -> List[int]: # Combine all sentences - if phoneme_input: - phonemes = list(unicodedata.normalize("NFD", text)) - else: - phonemes = [ - p - for sentence_phonemes in _PHONEMIZER.phonemize(voice, text) - for p in sentence_phonemes - ] + phonemes = [ + p + for sentence_phonemes in _PHONEMIZER.phonemize(voice, text) + for p in sentence_phonemes + ] if verbose is True: _LOGGER.debug("Phonemes: %s", phonemes) @@ -592,9 +512,6 @@ def main() -> int: type=int, help="Maximum number of speakers to use (default: no limit)", ) - parser.add_argument( - "--phoneme-input", action="store_true", help="Treat input text as phoneme input" - ) parser.add_argument("--verbose", action="store_true") args = parser.parse_args().__dict__ diff --git a/piper_sample_generator/impulses/Accoustic2_Impulse.wav b/impulses/Accoustic2_Impulse.wav similarity index 100% rename from piper_sample_generator/impulses/Accoustic2_Impulse.wav rename to impulses/Accoustic2_Impulse.wav diff --git a/piper_sample_generator/impulses/Blatty Plate.wav b/impulses/Blatty Plate.wav similarity index 100% rename from piper_sample_generator/impulses/Blatty Plate.wav rename to impulses/Blatty Plate.wav diff --git a/piper_sample_generator/impulses/Concrete Room.wav b/impulses/Concrete Room.wav similarity index 100% rename from piper_sample_generator/impulses/Concrete Room.wav rename to impulses/Concrete Room.wav diff --git a/piper_sample_generator/impulses/Derlon Sanctuary.wav b/impulses/Derlon Sanctuary.wav similarity index 100% rename from piper_sample_generator/impulses/Derlon Sanctuary.wav rename to impulses/Derlon Sanctuary.wav diff --git a/piper_sample_generator/impulses/Fat Bass.wav b/impulses/Fat Bass.wav similarity index 100% rename from piper_sample_generator/impulses/Fat Bass.wav rename to impulses/Fat Bass.wav diff --git a/piper_sample_generator/impulses/Reverse Gate.wav b/impulses/Reverse Gate.wav similarity index 100% rename from piper_sample_generator/impulses/Reverse Gate.wav rename to impulses/Reverse Gate.wav diff --git a/piper_sample_generator/impulses/Symphonic.wav b/impulses/Symphonic.wav similarity index 100% rename from piper_sample_generator/impulses/Symphonic.wav rename to impulses/Symphonic.wav diff --git a/piper_sample_generator/impulses/ir_bathroom1.wav b/impulses/ir_bathroom1.wav similarity index 100% rename from piper_sample_generator/impulses/ir_bathroom1.wav rename to impulses/ir_bathroom1.wav diff --git a/piper_sample_generator/__init__.py b/piper_sample_generator/__init__.py deleted file mode 100644 index 0142775..0000000 --- a/piper_sample_generator/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Piper sample generator.""" diff --git a/pylintrc b/pylintrc index 60fdb1d..561b2f1 100644 --- a/pylintrc +++ b/pylintrc @@ -1,6 +1,3 @@ -[MASTER] -ignored-modules=torch - [MESSAGES CONTROL] disable= format, @@ -34,7 +31,14 @@ disable= missing-class-docstring, missing-function-docstring, import-error, - relative-beyond-top-level + consider-using-with [FORMAT] expected-line-ending-format=LF + +[TYPECHECK] + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members=numpy.*,torch.* diff --git a/pyproject.toml b/pyproject.toml index 76cf7d2..5061a05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,31 +4,41 @@ build-backend = "setuptools.build_meta" [project] name = "piper-sample-generator" -version = "3.2.0" -license = {text = "MIT"} +version = "3.0.0" +license = {text = "Apache-2.0"} description = "Generate TTS audio samples for training wake word systems" readme = "README.md" authors = [ {name = "The Home Assistant Authors", email = "hello@home-assistant.io"} ] keywords = ["piper", "sample", "tts", "wakeword"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Topic :: Text Processing :: Linguistic", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] requires-python = ">=3.9.0" dependencies = [ - "audiomentations==0.33.0", - "piper-tts==1.3.0", - "numpy>=2,<3", + "piper-tts>=1.3.0,<2", "torch>=2,<3", "torchaudio", - "webrtcvad", + "audiomentations", + "numpy", ] [project.optional-dependencies] dev = [ - "black==22.12.0", - "flake8==6.0.0", - "isort==5.11.3", - "mypy==0.991", - "pylint==2.15.9", + "black==24.8.0", + "flake8==7.2.0", + "mypy==1.14.0", + "pylint==3.2.7", + "pytest==8.3.5", ] [project.urls] @@ -39,7 +49,4 @@ platforms = ["any"] zip-safe = true [tool.setuptools.packages.find] -include = ["piper_sample_generator*"] - -[tool.setuptools.package-data] -piper_sample_generator = ["impulses/*.wav"] +include = [] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8443d84 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +audiomentations==0.33.0 +piper-phonemize==1.1.0 +numpy<2 +torch<2 +torchaudio +webrtcvad diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..77190e6 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,5 @@ +black==22.12.0 +flake8==6.0.0 +isort==5.11.3 +mypy==0.991 +pylint==2.15.9 diff --git a/script/format b/script/format index b8b283b..7f04417 100755 --- a/script/format +++ b/script/format @@ -6,7 +6,7 @@ from pathlib import Path _DIR = Path(__file__).parent _PROGRAM_DIR = _DIR.parent _VENV_DIR = _PROGRAM_DIR / ".venv" -_MODULE_DIR = _PROGRAM_DIR / "piper_sample_generator" +_SCRIPT = _PROGRAM_DIR / "generate_samples.py" if _VENV_DIR.exists(): context = venv.EnvBuilder().ensure_directories(_VENV_DIR) @@ -14,5 +14,5 @@ if _VENV_DIR.exists(): else: python_exe = "python3" -subprocess.check_call([python_exe, "-m", "black", str(_MODULE_DIR)]) -subprocess.check_call([python_exe, "-m", "isort", str(_MODULE_DIR)]) +subprocess.check_call([python_exe, "-m", "black", str(_SCRIPT)]) +subprocess.check_call([python_exe, "-m", "isort", str(_SCRIPT)]) diff --git a/script/lint b/script/lint index 34222f0..e4231e0 100755 --- a/script/lint +++ b/script/lint @@ -6,7 +6,7 @@ from pathlib import Path _DIR = Path(__file__).parent _PROGRAM_DIR = _DIR.parent _VENV_DIR = _PROGRAM_DIR / ".venv" -_MODULE_DIR = _PROGRAM_DIR / "piper_sample_generator" +_SCRIPT = _PROGRAM_DIR / "generate_samples.py" if _VENV_DIR.exists(): context = venv.EnvBuilder().ensure_directories(_VENV_DIR) @@ -14,8 +14,8 @@ if _VENV_DIR.exists(): else: python_exe = "python3" -subprocess.check_call([python_exe, "-m", "black", str(_MODULE_DIR), "--check"]) -subprocess.check_call([python_exe, "-m", "isort", str(_MODULE_DIR), "--check"]) -subprocess.check_call([python_exe, "-m", "flake8", str(_MODULE_DIR)]) -subprocess.check_call([python_exe, "-m", "pylint", str(_MODULE_DIR)]) -subprocess.check_call([python_exe, "-m", "mypy", str(_MODULE_DIR)]) +subprocess.check_call([python_exe, "-m", "black", str(_SCRIPT), "--check"]) +subprocess.check_call([python_exe, "-m", "isort", str(_SCRIPT), "--check"]) +subprocess.check_call([python_exe, "-m", "flake8", str(_SCRIPT)]) +subprocess.check_call([python_exe, "-m", "pylint", str(_SCRIPT)]) +subprocess.check_call([python_exe, "-m", "mypy", str(_SCRIPT)]) diff --git a/script/run b/script/run index ae921c9..f2837d0 100755 --- a/script/run +++ b/script/run @@ -14,4 +14,4 @@ if _VENV_DIR.exists(): else: python_exe = "python3" -subprocess.check_call([python_exe, "-m", "piper_sample_generator"] + sys.argv[1:]) +subprocess.check_call([python_exe, "generate_samples.py"] + sys.argv[1:])