From a1f84c5ec37e5fff558251429cf65830411ac54f Mon Sep 17 00:00:00 2001 From: dscripka Date: Tue, 20 Jun 2023 07:35:39 -0400 Subject: [PATCH 1/7] Refactor to allow for usage of generation within Python script, added automatic resampling to 16khz --- generate_samples.py | 331 ++++++++++++++++++++++++++++---------------- 1 file changed, 208 insertions(+), 123 deletions(-) diff --git a/generate_samples.py b/generate_samples.py index f1f098d..b665a40 100755 --- a/generate_samples.py +++ b/generate_samples.py @@ -2,53 +2,77 @@ import argparse import itertools as it import json +import os import logging import unicodedata import wave from pathlib import Path +from tqdm import tqdm +from types import SimpleNamespace +from typing import Union, List import numpy as np import torch +import torchaudio from espeak_phonemizer import Phonemizer from piper_train.vits import commons _DIR = Path(__file__).parent _LOGGER = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) +# Main generation function +def generate_samples( + text: Union[List, str], + output_dir: str, + max_samples: int=None, + file_names: List[str] = [], + model: str = os.path.join(Path(__file__).parent, "models", "en-us-libritts-high.pt"), + batch_size: int = 1, + slerp_weights: List[float] = [0.5], + length_scales: List[float] = [0.75, 1, 1.25], + noise_scales: List[float] = [0.667], + noise_scale_ws: List[float] = [0.8], + max_speakers: float = None, + **kwargs + ) -> None: + """ + Generate synthetic speech clips, saving the clips to the specified outpur directory. -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("text") - parser.add_argument("--max-samples", required=True, type=int) - parser.add_argument("--model", default=_DIR / "models" / "en-us-libritts-high.pt") - parser.add_argument("--batch-size", type=int, default=1) - parser.add_argument("--slerp-weights", nargs="+", type=float, default=[0.5]) - parser.add_argument( - "--length-scales", nargs="+", type=float, default=[1.0, 0.75, 1.25] - ) - parser.add_argument("--noise-scales", nargs="+", type=float, default=[0.667]) - parser.add_argument("--noise-scale-ws", nargs="+", type=float, default=[0.8]) - parser.add_argument("--output-dir", default="output") - parser.add_argument( - "--max-speakers", - type=int, - help="Maximum number of speakers to use (default: all)", - ) - args = parser.parse_args() - logging.basicConfig(level=logging.DEBUG) + Args: + text (List[str]): The text to convert into speech. Can be either a + a list of strings, or a path to a file with text on each line. + output_dir (str): The location to save the generated clips. + max_samples (int): The maximum number of samples to generate. + file_names (List[str]): The names to use when saving the files. Must be the same length + as the `text` argument, if a list. + model (str): The path to the STT model to use for generation. + batch_size (int): The batch size to use when generated the clips + slerp_weights (List[float]): The weights to use when mixing speakers via SLERP. + length_scales (List[float]): Controls the average duration/speed of the generated speech. + noise_scales (List[float]): A parameter for overall variability of the generated speech. + noise_scale_ws (List[float]): A parameter for the stochastic duration of words/phonemes. + max_speakers (int): The maximum speaker number to use, if the model is multi-speaker. - _LOGGER.debug("Loading %s", args.model) - model_path = Path(args.model) + Returns: + None + """ + + if max_samples is None: + max_samples = len(text) + + _LOGGER.debug("Loading %s", model) + model_path = Path(model) model = torch.load(model_path) model.eval() - _LOGGER.info("Successfully loaded %s", args.model) + _LOGGER.info("Successfully loaded the model") if torch.cuda.is_available(): model.cuda() _LOGGER.debug("CUDA available, using GPU") - output_dir = Path(args.output_dir) + output_dir = Path(output_dir) output_dir.mkdir(parents=True, exist_ok=True) config_path = f"{model_path}.json" @@ -58,13 +82,146 @@ def main() -> None: voice = config["espeak"]["voice"] sample_rate = config["audio"]["sample_rate"] num_speakers = config["num_speakers"] - if args.max_speakers is not None: - num_speakers = min(num_speakers, args.max_speakers) + if max_speakers is not None: + num_speakers = min(num_speakers, max_speakers) phonemizer = Phonemizer(voice) - phonemes_str = phonemizer.phonemize(args.text) + + max_len = None + + sample_idx = 0 + is_done = False + settings_iter = it.cycle( + it.product( + slerp_weights, + length_scales, + noise_scales, + noise_scale_ws, + ) + ) + + # Define resampler to get to 16khz (https://pytorch.org/audio/stable/tutorials/audio_resampling_tutorial.html#kaiser-best) + sample_rate = 22050 + resample_rate = 16000 + resampler = torchaudio.transforms.Resample( + sample_rate, + resample_rate, + lowpass_filter_width=64, + rolloff=0.9475937167399596, + resampling_method="kaiser_window", + beta=14.769656459379492 + ) + + speakers_iter = it.cycle(it.product(range(num_speakers), range(num_speakers))) + speakers_batch = list(it.islice(speakers_iter, 0, batch_size)) + if isinstance(text, str) and os.path.exists(text): + texts = it.cycle([i.strip() for i in open(text, 'r').readlines() if len(i.strip()) > 0]) + elif isinstance(text, list): + texts = it.cycle(text) + else: + texts = it.cycle([text]) + + if file_names: + file_names = it.cycle(file_names) + + batch_idx = 0 + while speakers_batch: + if is_done: + break + + batch_size = len(speakers_batch) + # slerp_weight, length_scale, noise_scale, noise_scale_w = next(settings_iter) + slerp_weight = np.random.choice(slerp_weights) + length_scale = np.random.choice(length_scales) + noise_scale = np.random.choice(noise_scales) + noise_scale_w = np.random.choice(noise_scale_ws) + + with torch.no_grad(): + speaker_1 = torch.LongTensor([s[0] for s in speakers_batch]) + speaker_2 = torch.LongTensor([s[1] for s in speakers_batch]) + + phoneme_ids = [get_phonemes(phonemizer, config, next(texts))]*batch_size + audio = generate_audio(model, speaker_1, speaker_2, phoneme_ids, slerp_weight, noise_scale, noise_scale_w, length_scale, max_len) + + # Resample audio + audio = resampler(audio.cpu()).numpy() + + audio_int16 = audio_float_to_int16(audio) + for audio_idx in range(batch_size): + if isinstance(file_names, it.cycle): + wav_path = output_dir / next(file_names) + else: + wav_path = output_dir / f"{sample_idx}.wav" + with wave.open(str(wav_path), "wb") as wav_file: + wav_file.setframerate(resample_rate) + wav_file.setsampwidth(2) + wav_file.setnchannels(1) + wav_file.writeframes(audio_int16[audio_idx]) + + # print(wav_path) + + sample_idx += 1 + if sample_idx >= max_samples: + is_done = True + break + + print(f"Batch {batch_idx +1}/{max_samples//batch_size} complete", " "*200, end='\r') + + # Next batch + # _LOGGER.debug("Batch %s complete", batch_idx + 1) + speakers_batch = list(it.islice(speakers_iter, 0, batch_size)) + batch_idx += 1 + + # _LOGGER.info("Done") + +def generate_audio(model, speaker_1, speaker_2, phoneme_ids, slerp_weight, noise_scale, noise_scale_w, length_scale, max_len): + x = torch.LongTensor(phoneme_ids)#.repeat((batch_size, 1)) + x_lengths = torch.LongTensor([len(i) for i in phoneme_ids])#.repeat(batch_size) + + if torch.cuda.is_available(): + speaker_1 = speaker_1.cuda() + speaker_2 = speaker_2.cuda() + x = x.cuda() + x_lengths = x_lengths.cuda() + + x, m_p_orig, logs_p_orig, x_mask = model.enc_p(x, x_lengths) + emb0 = model.emb_g(speaker_1) + emb1 = model.emb_g(speaker_2) + g = slerp(emb0, emb1, slerp_weight).unsqueeze(-1) # [b, h, 1] + + if model.use_sdp: + logw = model.dp(x, x_mask, g=g, reverse=True, noise_scale=noise_scale_w) + else: + logw = model.dp(x, x_mask, g=g) + w = torch.exp(logw) * x_mask * length_scale + w_ceil = torch.ceil(w) + y_lengths = torch.clamp_min(torch.sum(w_ceil, [1, 2]), 1).long() + y_mask = torch.unsqueeze( + commons.sequence_mask(y_lengths, y_lengths.max()), 1 + ).type_as(x_mask) + attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1) + attn = commons.generate_path(w_ceil, attn_mask) + + m_p = torch.matmul(attn.squeeze(1), m_p_orig.transpose(1, 2)).transpose( + 1, 2 + ) # [b, t', t], [b, t, d] -> [b, d, t'] + logs_p = torch.matmul( + attn.squeeze(1), logs_p_orig.transpose(1, 2) + ).transpose( + 1, 2 + ) # [b, t', t], [b, t, d] -> [b, d, t'] + + z_p = m_p + torch.randn_like(m_p) * torch.exp(logs_p) * noise_scale + z = model.flow(z_p, y_mask, g=g, reverse=True) + o = model.dec((z * y_mask)[:, :, :max_len], g=g) + + audio = o + return audio + +def get_phonemes(phonemizer, config, text): + phonemes_str = phonemizer.phonemize(text) phonemes = list(unicodedata.normalize("NFD", phonemes_str)) - _LOGGER.debug("Phonemes: %s", phonemes) + # _LOGGER.debug("Phonemes: %s", phonemes) id_map = config["phoneme_id_map"] phoneme_ids = list(id_map["^"]) @@ -75,100 +232,7 @@ def main() -> None: phoneme_ids.extend(id_map["_"]) phoneme_ids.extend(id_map["$"]) - _LOGGER.debug("Phonemes ids: %s", phoneme_ids) - - max_len = None - - sample_idx = 0 - is_done = False - settings_iter = it.cycle( - it.product( - args.slerp_weights, - args.length_scales, - args.noise_scales, - args.noise_scale_ws, - ) - ) - - speakers_iter = it.cycle(it.product(range(num_speakers), range(num_speakers))) - speakers_batch = list(it.islice(speakers_iter, 0, args.batch_size)) - batch_idx = 0 - while speakers_batch: - if is_done: - break - - batch_size = len(speakers_batch) - slerp_weight, length_scale, noise_scale, noise_scale_w = next(settings_iter) - - with torch.no_grad(): - speaker_1 = torch.LongTensor([s[0] for s in speakers_batch]) - speaker_2 = torch.LongTensor([s[1] for s in speakers_batch]) - - x = torch.LongTensor(phoneme_ids).repeat((batch_size, 1)) - x_lengths = torch.LongTensor([len(phoneme_ids)]).repeat(batch_size) - - if torch.cuda.is_available(): - speaker_1 = speaker_1.cuda() - speaker_2 = speaker_2.cuda() - x = x.cuda() - x_lengths = x_lengths.cuda() - - x, m_p_orig, logs_p_orig, x_mask = model.enc_p(x, x_lengths) - emb0 = model.emb_g(speaker_1) - emb1 = model.emb_g(speaker_2) - g = slerp(emb0, emb1, slerp_weight).unsqueeze(-1) # [b, h, 1] - - if model.use_sdp: - logw = model.dp(x, x_mask, g=g, reverse=True, noise_scale=noise_scale_w) - else: - logw = model.dp(x, x_mask, g=g) - w = torch.exp(logw) * x_mask * length_scale - w_ceil = torch.ceil(w) - y_lengths = torch.clamp_min(torch.sum(w_ceil, [1, 2]), 1).long() - y_mask = torch.unsqueeze( - commons.sequence_mask(y_lengths, y_lengths.max()), 1 - ).type_as(x_mask) - attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1) - attn = commons.generate_path(w_ceil, attn_mask) - - m_p = torch.matmul(attn.squeeze(1), m_p_orig.transpose(1, 2)).transpose( - 1, 2 - ) # [b, t', t], [b, t, d] -> [b, d, t'] - logs_p = torch.matmul( - attn.squeeze(1), logs_p_orig.transpose(1, 2) - ).transpose( - 1, 2 - ) # [b, t', t], [b, t, d] -> [b, d, t'] - - z_p = m_p + torch.randn_like(m_p) * torch.exp(logs_p) * noise_scale - z = model.flow(z_p, y_mask, g=g, reverse=True) - o = model.dec((z * y_mask)[:, :, :max_len], g=g) - - audio = o.cpu().numpy() - - audio_int16 = audio_float_to_int16(audio) - for audio_idx in range(batch_size): - wav_path = output_dir / f"{sample_idx}.wav" - with wave.open(str(wav_path), "wb") as wav_file: - wav_file.setframerate(sample_rate) - wav_file.setsampwidth(2) - wav_file.setnchannels(1) - wav_file.writeframes(audio_int16[audio_idx]) - - print(wav_path) - - sample_idx += 1 - if sample_idx >= args.max_samples: - is_done = True - break - - # Next batch - _LOGGER.debug("Batch %s complete", batch_idx + 1) - speakers_batch = list(it.islice(speakers_iter, 0, args.batch_size)) - batch_idx += 1 - - _LOGGER.info("Done") - + return phoneme_ids def slerp(v1, v2, t, DOT_THR=0.9995, zdim=-1): """SLERP for pytorch tensors interpolating `v1` to `v2` with scale of `t`. @@ -226,4 +290,25 @@ def audio_float_to_int16( if __name__ == "__main__": - main() + # Get command line arguments + parser = argparse.ArgumentParser() + parser.add_argument("text") + parser.add_argument("--max-samples", required=True, type=int) + parser.add_argument("--model", default=_DIR / "models" / "en-us-libritts-high.pt") + parser.add_argument("--batch-size", type=int, default=1) + parser.add_argument("--slerp-weights", nargs="+", type=float, default=[0.5]) + parser.add_argument( + "--length-scales", nargs="+", type=float, default=[1.0, 0.75, 1.25, 1.4] + ) + parser.add_argument("--noise-scales", nargs="+", type=float, default=[0.667, .75, .85, 0.9, 1.0, 1.4]) + parser.add_argument("--noise-scale-ws", nargs="+", type=float, default=[0.8]) + parser.add_argument("--output-dir", default="output") + parser.add_argument( + "--max-speakers", + type=int, + help="Maximum number of speakers to use (default: all)", + ) + args = parser.parse_args().__dict__ + + # Generate speech + generate_samples(**args) From 8b4b20b7122bde97ad47bc482b3ea402cf4e8dc6 Mon Sep 17 00:00:00 2001 From: dscripka Date: Sun, 6 Aug 2023 11:09:39 -0400 Subject: [PATCH 2/7] Cleanup and verbose argument --- generate_samples.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/generate_samples.py b/generate_samples.py index b665a40..bac7131 100755 --- a/generate_samples.py +++ b/generate_samples.py @@ -35,10 +35,11 @@ def generate_samples( noise_scales: List[float] = [0.667], noise_scale_ws: List[float] = [0.8], max_speakers: float = None, + verbose: bool = False, **kwargs ) -> None: """ - Generate synthetic speech clips, saving the clips to the specified outpur directory. + Generate synthetic speech clips, saving the clips to the specified output directory. Args: text (List[str]): The text to convert into speech. Can be either a @@ -54,6 +55,7 @@ def generate_samples( noise_scales (List[float]): A parameter for overall variability of the generated speech. noise_scale_ws (List[float]): A parameter for the stochastic duration of words/phonemes. max_speakers (int): The maximum speaker number to use, if the model is multi-speaker. + verbose (bool): Enable or disable more detailed logging messages (default: False). Returns: None @@ -130,17 +132,13 @@ def generate_samples( break batch_size = len(speakers_batch) - # slerp_weight, length_scale, noise_scale, noise_scale_w = next(settings_iter) - slerp_weight = np.random.choice(slerp_weights) - length_scale = np.random.choice(length_scales) - noise_scale = np.random.choice(noise_scales) - noise_scale_w = np.random.choice(noise_scale_ws) + slerp_weight, length_scale, noise_scale, noise_scale_w = next(settings_iter) with torch.no_grad(): speaker_1 = torch.LongTensor([s[0] for s in speakers_batch]) speaker_2 = torch.LongTensor([s[1] for s in speakers_batch]) - phoneme_ids = [get_phonemes(phonemizer, config, next(texts))]*batch_size + phoneme_ids = [get_phonemes(phonemizer, config, next(texts), verbose)]*batch_size audio = generate_audio(model, speaker_1, speaker_2, phoneme_ids, slerp_weight, noise_scale, noise_scale_w, length_scale, max_len) # Resample audio @@ -165,18 +163,18 @@ def generate_samples( is_done = True break - print(f"Batch {batch_idx +1}/{max_samples//batch_size} complete", " "*200, end='\r') + # print(f"Batch {batch_idx +1}/{max_samples//batch_size} complete", " "*200, end='\r') # Next batch - # _LOGGER.debug("Batch %s complete", batch_idx + 1) + _LOGGER.debug(f"Batch {batch_idx +1}/{max_samples//batch_size} complete") speakers_batch = list(it.islice(speakers_iter, 0, batch_size)) batch_idx += 1 - # _LOGGER.info("Done") + _LOGGER.info("Done") def generate_audio(model, speaker_1, speaker_2, phoneme_ids, slerp_weight, noise_scale, noise_scale_w, length_scale, max_len): - x = torch.LongTensor(phoneme_ids)#.repeat((batch_size, 1)) - x_lengths = torch.LongTensor([len(i) for i in phoneme_ids])#.repeat(batch_size) + x = torch.LongTensor(phoneme_ids) + x_lengths = torch.LongTensor([len(i) for i in phoneme_ids]) if torch.cuda.is_available(): speaker_1 = speaker_1.cuda() @@ -218,10 +216,11 @@ def generate_audio(model, speaker_1, speaker_2, phoneme_ids, slerp_weight, noise audio = o return audio -def get_phonemes(phonemizer, config, text): +def get_phonemes(phonemizer, config, text, verbose): phonemes_str = phonemizer.phonemize(text) phonemes = list(unicodedata.normalize("NFD", phonemes_str)) - # _LOGGER.debug("Phonemes: %s", phonemes) + if verbose is True: + _LOGGER.debug("Phonemes: %s", phonemes) id_map = config["phoneme_id_map"] phoneme_ids = list(id_map["^"]) From f82afc6596785f6df51c81bcf059689c49dd8232 Mon Sep 17 00:00:00 2001 From: dscripka Date: Sun, 6 Aug 2023 11:19:56 -0400 Subject: [PATCH 3/7] Added example of import usage to README --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index aaab83d..02bc367 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ wget -O models/en-us-libritts-high.pt 'https://github.com/rhasspy/piper-sample-g ## Run -Generate a small set of samples: +Generate a small set of samples with the CLI: ``` sh python3 generate_samples.py 'okay, piper.' --max-samples 10 --output-dir okay_piper/ @@ -46,6 +46,16 @@ Setting `--max-speakers` to a value less than 904 (the number if LibriTTS) is re See `--help` for more options, including adjust the `--length-scales` (speaking speeds) and `--slerp-weights` (speaker blending) which are cycled per batch. +Alternatively, you can import the generate function into a another Python script: + +```python +from generate_samples import generate_samples # make sure to add this to your Python path as needed + +generate_samples(text = ["okay, piper"], max_samples = 100, output_dir = output_dir, batch_size=10) +``` + +There are some additional arguments available when importing the function directly, see the docstring of `generate_sample` for more information. + ### Augmentation Once you have samples generating, you can augment them using [audiomentation](https://iver56.github.io/audiomentations/): From 8544faa48ceb7eea1be62b0cd1d91e8c2d9d1735 Mon Sep 17 00:00:00 2001 From: dscripka Date: Sun, 6 Aug 2023 11:20:34 -0400 Subject: [PATCH 4/7] typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 02bc367..7537244 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Setting `--max-speakers` to a value less than 904 (the number if LibriTTS) is re See `--help` for more options, including adjust the `--length-scales` (speaking speeds) and `--slerp-weights` (speaker blending) which are cycled per batch. -Alternatively, you can import the generate function into a another Python script: +Alternatively, you can import the generate function into another Python script: ```python from generate_samples import generate_samples # make sure to add this to your Python path as needed From 28979b89642c127fb70e8fcc9dfa4bf8ee7e4738 Mon Sep 17 00:00:00 2001 From: dscripka Date: Wed, 16 Aug 2023 21:44:51 -0400 Subject: [PATCH 5/7] Added functionality in 'generate_samples' function to attempt to auto-recover from CUDA OOM errors --- generate_samples.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/generate_samples.py b/generate_samples.py index bac7131..1577bde 100755 --- a/generate_samples.py +++ b/generate_samples.py @@ -3,6 +3,7 @@ import argparse import itertools as it import json import os +import gc import logging import unicodedata import wave @@ -36,6 +37,7 @@ def generate_samples( noise_scale_ws: List[float] = [0.8], max_speakers: float = None, verbose: bool = False, + auto_reduce_batch_size: bool = False, **kwargs ) -> None: """ @@ -56,6 +58,8 @@ def generate_samples( noise_scale_ws (List[float]): A parameter for the stochastic duration of words/phonemes. max_speakers (int): The maximum speaker number to use, if the model is multi-speaker. verbose (bool): Enable or disable more detailed logging messages (default: False). + auto_reduce_batch_size (bool): Automatically and temporarily reduce the batch size + if CUDA OOM errors are detected, and try to resume generation. Returns: None @@ -139,13 +143,26 @@ def generate_samples( speaker_2 = torch.LongTensor([s[1] for s in speakers_batch]) phoneme_ids = [get_phonemes(phonemizer, config, next(texts), verbose)]*batch_size - audio = generate_audio(model, speaker_1, speaker_2, phoneme_ids, slerp_weight, noise_scale, noise_scale_w, length_scale, max_len) + if auto_reduce_batch_size: + oom_error = True + counter = 1 + while oom_error is True: + try: + audio = generate_audio(model, speaker_1[0:batch_size//counter], speaker_2[0:batch_size//counter], phoneme_ids[0:batch_size//counter], + slerp_weight, noise_scale, noise_scale_w, length_scale, max_len) + oom_error = False + except torch.cuda.OutOfMemoryError: + torch.cuda.empty_cache() + gc.collect() + counter += 1 # reduce batch size to avoid OOM errors + else: + audio = generate_audio(model, speaker_1, speaker_2, phoneme_ids, slerp_weight, noise_scale, noise_scale_w, length_scale, max_len) # Resample audio audio = resampler(audio.cpu()).numpy() audio_int16 = audio_float_to_int16(audio) - for audio_idx in range(batch_size): + for audio_idx in range(audio_int16.shape[0]): if isinstance(file_names, it.cycle): wav_path = output_dir / next(file_names) else: From b104c2412dc224d2d6c4cbe9359204c149180865 Mon Sep 17 00:00:00 2001 From: dscripka Date: Sun, 10 Sep 2023 15:11:45 -0400 Subject: [PATCH 6/7] Fixed major bug where batch sizes >= 1 would not generate all input texts --- generate_samples.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/generate_samples.py b/generate_samples.py index 1577bde..1947ecd 100755 --- a/generate_samples.py +++ b/generate_samples.py @@ -142,7 +142,18 @@ def generate_samples( speaker_1 = torch.LongTensor([s[0] for s in speakers_batch]) speaker_2 = torch.LongTensor([s[1] for s in speakers_batch]) - phoneme_ids = [get_phonemes(phonemizer, config, next(texts), verbose)]*batch_size + phoneme_ids = [get_phonemes(phonemizer, config, next(texts), verbose) for i in range(batch_size)] + + def right_pad_lists(lists): + max_length = max(len(l) for l in lists) + padded_lists = [] + for l in lists: + padded_l = l + [1] * (max_length - len(l)) # phoneme 1 (corresponding to '^' character seems to work best) + padded_lists.append(padded_l) + return padded_lists + + phoneme_ids = right_pad_lists(phoneme_ids) + if auto_reduce_batch_size: oom_error = True counter = 1 From f1988a4d54eddb23d99e86f0adfef6226a85acc7 Mon Sep 17 00:00:00 2001 From: dscripka Date: Mon, 11 Sep 2023 07:56:17 -0400 Subject: [PATCH 7/7] Added use of pywebrtc to remove excessive silence from generated clips --- generate_samples.py | 21 ++++++++++++++++++--- requirements.txt | 1 + 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/generate_samples.py b/generate_samples.py index 1947ecd..c11184e 100755 --- a/generate_samples.py +++ b/generate_samples.py @@ -11,6 +11,7 @@ from pathlib import Path from tqdm import tqdm from types import SimpleNamespace from typing import Union, List +import webrtcvad import numpy as np import torch @@ -174,6 +175,9 @@ def generate_samples( audio_int16 = audio_float_to_int16(audio) for audio_idx in range(audio_int16.shape[0]): + # Use webrtcvad to trip silence from the clips + audio_data = remove_silence(audio_int16[audio_idx].flatten())[None,] + if isinstance(file_names, it.cycle): wav_path = output_dir / next(file_names) else: @@ -182,9 +186,7 @@ def generate_samples( wav_file.setframerate(resample_rate) wav_file.setsampwidth(2) wav_file.setnchannels(1) - wav_file.writeframes(audio_int16[audio_idx]) - - # print(wav_path) + wav_file.writeframes(audio_data) sample_idx += 1 if sample_idx >= max_samples: @@ -200,6 +202,19 @@ def generate_samples( _LOGGER.info("Done") +def remove_silence(x, frame_duration=.030, sample_rate=16000, min_start = 2000): + """Uses webrtc voice activity detection to remove silence from the clips""" + vad = webrtcvad.Vad(0) + if x.dtype == np.float32 or x.dtype == np.float64: + x = (x*32767).astype(np.int16) + x_new = x[0:min_start].tolist() + step_size = int(sample_rate*frame_duration) + for i in range(min_start, x.shape[0] - step_size, step_size): + vad_res = vad.is_speech(x[i:i+step_size].tobytes(), sample_rate) + if vad_res: + x_new.extend(x[i:i+step_size].tolist()) + return np.array(x_new).astype(np.int16) + def generate_audio(model, speaker_1, speaker_2, phoneme_ids, slerp_weight, noise_scale, noise_scale_w, length_scale, max_len): x = torch.LongTensor(phoneme_ids) x_lengths = torch.LongTensor([len(i) for i in phoneme_ids]) diff --git a/requirements.txt b/requirements.txt index fc244b8..17c5857 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ audiomentations espeak-phonemizer numpy torch +webrtcvad \ No newline at end of file