Update models (Rebased)

.pylintrc

@@ -168,7 +168,8 @@ disable=missing-docstring,
  exception-escape,
  comprehension-escape,
  duplicate-code,
- not-callable
+ not-callable,
+ import-outside-toplevel
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

TTS/bin/extract_tts_spectrograms.py

@@ -13,28 +13,28 @@
 from TTS.tts.datasets import TTSDataset, load_tts_samples
 from TTS.tts.models import setup_model
 from TTS.tts.utils.speakers import SpeakerManager
+from TTS.tts.utils.text.tokenizer import TTSTokenizer
 from TTS.utils.audio import AudioProcessor
 from TTS.utils.generic_utils import count_parameters
 
 use_cuda = torch.cuda.is_available()
 
 
 def setup_loader(ap, r, verbose=False):
+ tokenizer, _ = TTSTokenizer.init_from_config(c)
  dataset = TTSDataset(
- r,
- c.text_cleaner,
+ outputs_per_step=r,
  compute_linear_spec=False,
- meta_data=meta_data,
+ samples=meta_data,
+ tokenizer=tokenizer,
  ap=ap,
- characters=c.characters if "characters" in c.keys() else None,
- add_blank=c["add_blank"] if "add_blank" in c.keys() else False,
  batch_group_size=0,
- min_seq_len=c.min_seq_len,
- max_seq_len=c.max_seq_len,
+ min_text_len=c.min_text_len,
+ max_text_len=c.max_text_len,
+ min_audio_len=c.min_audio_len,
+ max_audio_len=c.max_audio_len,
  phoneme_cache_path=c.phoneme_cache_path,
- use_phonemes=c.use_phonemes,
- phoneme_language=c.phoneme_language,
- enable_eos_bos=c.enable_eos_bos_chars,
+ precompute_num_workers=0,
  use_noise_augment=False,
  verbose=verbose,
  speaker_id_mapping=speaker_manager.speaker_ids if c.use_speaker_embedding else None,
@@ -44,7 +44,7 @@ def setup_loader(ap, r, verbose=False):
  if c.use_phonemes and c.compute_input_seq_cache:
  # precompute phonemes to have a better estimate of sequence lengths.
  dataset.compute_input_seq(c.num_loader_workers)
- dataset.sort_and_filter_items(c.get("sort_by_audio_len", default=False))
+ dataset.preprocess_samples()
 
  loader = DataLoader(
  dataset,
@@ -75,8 +75,8 @@ def set_filename(wav_path, out_path):
 
 def format_data(data):
  # setup input data
- text_input = data["text"]
- text_lengths = data["text_lengths"]
+ text_input = data["token_id"]
+ text_lengths = data["token_id_lengths"]
  mel_input = data["mel"]
  mel_lengths = data["mel_lengths"]
  item_idx = data["item_idxs"]

TTS/bin/find_unique_phonemes.py

@@ -7,14 +7,15 @@
 
 from TTS.config import load_config
 from TTS.tts.datasets import load_tts_samples
-from TTS.tts.utils.text import text2phone
+from TTS.tts.utils.text.phonemizers.gruut_wrapper import Gruut
+
+phonemizer = Gruut(language="en-us")
 
 
 def compute_phonemes(item):
  try:
  text = item[0]
- language = item[-1]
- ph = text2phone(text, language, use_espeak_phonemes=c.use_espeak_phonemes).split("|")
+ ph = phonemizer.phonemize(text).split("|")
  except:
  return []
  return list(set(ph))

TTS/bin/resample.py

@@ -26,6 +26,7 @@ def resample_file(func_args):
  --input_dir /root/LJSpeech-1.1/
  --output_sr 22050
  --output_dir /root/resampled_LJSpeech-1.1/
+ --file_ext wav
  --n_jobs 24
  """,
  formatter_class=RawTextHelpFormatter,
@@ -55,6 +56,14 @@ def resample_file(func_args):
  help="Path of the destination folder. If not defined, the operation is done in place",
  )
 
+ parser.add_argument(
+ "--file_ext",
+ type=str,
+ default="wav",
+ required=False,
+ help="Extension of the audio files to resample",
+ )
+
  parser.add_argument(
  "--n_jobs", type=int, default=None, help="Number of threads to use, by default it uses all cores"
  )
@@ -67,7 +76,7 @@ def resample_file(func_args):
  args.input_dir = args.output_dir
 
  print("Resampling the audio files...")
- audio_files = glob.glob(os.path.join(args.input_dir, "**/*.wav"), recursive=True)
+ audio_files = glob.glob(os.path.join(args.input_dir, f"**/*.{args.file_ext}"), recursive=True)
  print(f"Found {len(audio_files)} files...")
  audio_files = list(zip(audio_files, len(audio_files) * [args.output_sr]))
  with Pool(processes=args.n_jobs) as p:

TTS/bin/train_tts.py

@@ -1,12 +1,9 @@
 import os
 
-from TTS.config import check_config_and_model_args, get_from_config_or_model_args, load_config, register_config
+from TTS.config import load_config, register_config
 from TTS.trainer import Trainer, TrainingArgs
 from TTS.tts.datasets import load_tts_samples
 from TTS.tts.models import setup_model
-from TTS.tts.utils.languages import LanguageManager
-from TTS.tts.utils.speakers import SpeakerManager
-from TTS.utils.audio import AudioProcessor
 
 
 def main():
@@ -42,36 +39,8 @@ def main():
  # load training samples
  train_samples, eval_samples = load_tts_samples(config.datasets, eval_split=True)
 
- # setup audio processor
- ap = AudioProcessor(**config.audio)
-
- # init speaker manager
- if check_config_and_model_args(config, "use_speaker_embedding", True):
- speaker_manager = SpeakerManager(data_items=train_samples + eval_samples)
- if hasattr(config, "model_args"):
- config.model_args.num_speakers = speaker_manager.num_speakers
- else:
- config.num_speakers = speaker_manager.num_speakers
- elif check_config_and_model_args(config, "use_d_vector_file", True):
- speaker_manager = SpeakerManager(d_vectors_file_path=get_from_config_or_model_args(config, "d_vector_file"))
- if hasattr(config, "model_args"):
- config.model_args.num_speakers = speaker_manager.num_speakers
- else:
- config.num_speakers = speaker_manager.num_speakers
- else:
- speaker_manager = None
-
- if hasattr(config, "use_language_embedding") and config.use_language_embedding:
- language_manager = LanguageManager(config=config)
- if hasattr(config, "model_args"):
- config.model_args.num_languages = language_manager.num_languages
- else:
- config.num_languages = language_manager.num_languages
- else:
- language_manager = None
-
  # init the model from config
- model = setup_model(config, speaker_manager, language_manager)
+ model = setup_model(config, train_samples + eval_samples)
 
  # init the trainer and 🚀
  trainer = Trainer(
@@ -81,7 +50,6 @@ def main():
  model=model,
  train_samples=train_samples,
  eval_samples=eval_samples,
- training_assets={"audio_processor": ap},
  parse_command_line_args=False,
  )
  trainer.fit()

TTS/config/shared_configs.py

@@ -57,6 +57,12 @@ class BaseAudioConfig(Coqpit):
  do_amp_to_db_mel (bool, optional):
  enable/disable amplitude to dB conversion of mel spectrograms. Defaults to True.
 
+ pitch_fmax (float, optional):
+ Maximum frequency of the F0 frames. Defaults to ```640```.
+
+ pitch_fmin (float, optional):
+ Minimum frequency of the F0 frames. Defaults to ```0```.
+
  trim_db (int):
  Silence threshold used for silence trimming. Defaults to 45.
 
@@ -135,6 +141,9 @@ class BaseAudioConfig(Coqpit):
  spec_gain: int = 20
  do_amp_to_db_linear: bool = True
  do_amp_to_db_mel: bool = True
+ # f0 params
+ pitch_fmax: float = 640.0
+ pitch_fmin: float = 0.0
  # normalization params
  signal_norm: bool = True
  min_level_db: int = -100

TTS/tts/configs/fast_pitch_config.py

@@ -89,12 +89,9 @@ class FastPitchConfig(BaseTTSConfig):
  pitch_loss_alpha (float):
  Weight for the pitch predictor's loss. If set 0, disables the pitch predictor. Defaults to 1.0.
 
- binary_loss_alpha (float):
+ binary_align_loss_alpha (float):
  Weight for the binary loss. If set 0, disables the binary loss. Defaults to 1.0.
 
- binary_align_loss_start_step (int):
- Start binary alignment loss after this many steps. Defaults to 20000.
-
  min_seq_len (int):
  Minimum input sequence length to be used at training.
 
@@ -129,12 +126,12 @@ class FastPitchConfig(BaseTTSConfig):
  duration_loss_type: str = "mse"
  use_ssim_loss: bool = True
  ssim_loss_alpha: float = 1.0
- dur_loss_alpha: float = 1.0
  spec_loss_alpha: float = 1.0
- pitch_loss_alpha: float = 1.0
  aligner_loss_alpha: float = 1.0
- binary_align_loss_alpha: float = 1.0
- binary_align_loss_start_step: int = 20000
+ pitch_loss_alpha: float = 0.1
+ dur_loss_alpha: float = 0.1
+ binary_align_loss_alpha: float = 0.1
+ binary_loss_warmup_epochs: int = 150
 
  # overrides
  min_seq_len: int = 13

TTS/tts/configs/glow_tts_config.py

@@ -153,6 +153,7 @@ class GlowTTSConfig(BaseTTSConfig):
 
  # multi-speaker settings
  use_speaker_embedding: bool = False
+ speakers_file: str = None
  use_d_vector_file: bool = False
  d_vector_file: str = False
 

TTS/tts/configs/shared_configs.py

@@ -53,6 +53,10 @@ class CharactersConfig(Coqpit):
  """Defines arguments for the `BaseCharacters` and its subclasses.
 
  Args:
+ characters_class (str):
+ Defines the class of the characters used. If None, we pick ```Phonemes``` or ```Graphemes``` based on
+ the configuration. Defaults to None.
+
  pad (str):
  characters in place of empty padding. Defaults to None.
 
@@ -78,12 +82,13 @@ class CharactersConfig(Coqpit):
 
  is_unique (bool):
  remove any duplicate characters in the character lists. It is a bandaid for compatibility with the old
- models trained with character lists with duplicates.
+ models trained with character lists with duplicates. Defaults to True.
 
  is_sorted (bool):
  Sort the characters in alphabetical order. Defaults to True.
  """
 
+ characters_class: str = None
  pad: str = None
  eos: str = None
  bos: str = None
@@ -166,9 +171,16 @@ class BaseTTSConfig(BaseTrainingConfig):
  compute_linear_spec (bool):
  If True data loader computes and returns linear spectrograms alongside the other data.
 
+ precompute_num_workers (int):
+ Number of workers to precompute features. Defaults to 0.
+
  use_noise_augment (bool):
  Augment the input audio with random noise.
 
+ start_by_longest (bool):
+ If True, the data loader will start loading the longest batch first. It is useful for checking OOM issues.
+ Defaults to False.
+
  add_blank (bool):
  Add blank characters between each other two characters. It improves performance for some models at expense
  of slower run-time due to the longer input sequence.
@@ -207,6 +219,7 @@ class BaseTTSConfig(BaseTrainingConfig):
  phoneme_cache_path: str = None
  # vocabulary parameters
  characters: CharactersConfig = None
+ add_blank: bool = False
  # training params
  batch_group_size: int = 0
  loss_masking: bool = None
@@ -218,8 +231,9 @@ class BaseTTSConfig(BaseTrainingConfig):
  max_text_len: int = float("inf")
  compute_f0: bool = False
  compute_linear_spec: bool = False
+ precompute_num_workers: int = 0
  use_noise_augment: bool = False
- add_blank: bool = False
+ start_by_longest: bool = False
  # dataset
  datasets: List[BaseDatasetConfig] = field(default_factory=lambda: [BaseDatasetConfig()])
  # optimizer

TTS/tts/configs/vits_config.py

@@ -67,15 +67,6 @@ class VitsConfig(BaseTTSConfig):
  compute_linear_spec (bool):
  If true, the linear spectrogram is computed and returned alongside the mel output. Do not change. Defaults to `True`.
 
- sort_by_audio_len (bool):
- If true, dataloder sorts the data by audio length else sorts by the input text length. Defaults to `True`.
-
- min_seq_len (int):
- Minimum sequnce length to be considered for training. Defaults to `0`.
-
- max_seq_len (int):
- Maximum sequnce length to be considered for training. Defaults to `500000`.
-
  r (int):
  Number of spectrogram frames to be generated at a time. Do not change. Defaults to `1`.
 
@@ -123,16 +114,14 @@ class VitsConfig(BaseTTSConfig):
  feat_loss_alpha: float = 1.0
  mel_loss_alpha: float = 45.0
  dur_loss_alpha: float = 1.0
+ aligner_loss_alpha = 1.0
  speaker_encoder_loss_alpha: float = 1.0
 
  # data loader params
  return_wav: bool = True
  compute_linear_spec: bool = True
 
  # overrides
- sort_by_audio_len: bool = True
- min_seq_len: int = 0
- max_seq_len: int = 500000
  r: int = 1 # DO NOT CHANGE
  add_blank: bool = True
 

TTS/tts/datasets/__init__.py

@@ -13,7 +13,7 @@ def split_dataset(items):
  """Split a dataset into train and eval. Consider speaker distribution in multi-speaker training.
 
  Args:
- items (List[List]): A list of samples. Each sample is a list of `[audio_path, text, speaker_id]`.
+ items (List[List]): A list of samples. Each sample is a list of `[text, audio_path, speaker_id]`.
  """
  speakers = [item[-1] for item in items]
  is_multi_speaker = len(set(speakers)) > 1
@@ -52,7 +52,7 @@ def load_tts_samples(
 
  formatter (Callable, optional): The preprocessing function to be applied to create the list of samples. It
  must take the root_path and the meta_file name and return a list of samples in the format of
- `[[audio_path, text, speaker_id], ...]]`. See the available formatters in `TTS.tts.dataset.formatter` as
+ `[[text, audio_path, speaker_id], ...]]`. See the available formatters in `TTS.tts.dataset.formatter` as
  example. Defaults to None.
 
  Returns: