2024-10-30 13:45:29.755 | INFO     | everyvoice.base_cli.helpers:save_configuration_to_log_dir:171 - Configuration
{
   "contact": {
      "contact_name": "Marc Tessier",
      "contact_email": "Marc.Tessier@nrc.gc.ca"
   },
   "model": {
      "encoder": {
         "layers": 4,
         "heads": 2,
         "input_dim": 256,
         "feedforward_dim": 1024,
         "conv_kernel_size": 9,
         "dropout": 0.2
      },
      "decoder": {
         "layers": 4,
         "heads": 2,
         "input_dim": 256,
         "feedforward_dim": 1024,
         "conv_kernel_size": 9,
         "dropout": 0.2
      },
      "variance_predictors": {
         "energy": {
            "loss": "mse",
            "n_layers": 5,
            "kernel_size": 3,
            "dropout": 0.5,
            "input_dim": 256,
            "n_bins": 256,
            "depthwise": true,
            "level": "phone"
         },
         "duration": {
            "loss": "mse",
            "n_layers": 5,
            "kernel_size": 3,
            "dropout": 0.5,
            "input_dim": 256,
            "n_bins": 256,
            "depthwise": true
         },
         "pitch": {
            "loss": "mse",
            "n_layers": 5,
            "kernel_size": 3,
            "dropout": 0.5,
            "input_dim": 256,
            "n_bins": 256,
            "depthwise": true,
            "level": "phone"
         }
      },
      "target_text_representation_level": "characters",
      "learn_alignment": true,
      "max_length": 1000,
      "mel_loss": "mse",
      "use_postnet": true,
      "multilingual": false,
      "multispeaker": false
   },
   "path_to_model_config_file": null,
   "training": {
      "batch_size": 16,
      "save_top_k_ckpts": 5,
      "ckpt_steps": null,
      "ckpt_epochs": 1,
      "val_check_interval": 500,
      "check_val_every_n_epoch": null,
      "max_epochs": 1000,
      "max_steps": 100000,
      "finetune_checkpoint": null,
      "training_filelist": "/gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/LJSpeech-1.1/ap-513/preprocessed/training_filelist.psv",
      "validation_filelist": "/gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/LJSpeech-1.1/ap-513/preprocessed/validation_filelist.psv",
      "filelist_loader": "everyvoice.utils.generic_dict_loader",
      "logger": {
         "name": "FeaturePredictionExperiment",
         "save_dir": "/gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/LJSpeech-1.1/ap-513/logs_and_checkpoints",
         "sub_dir_callable": "everyvoice.utils.get_current_time",
         "version": "base"
      },
      "val_data_workers": 0,
      "train_data_workers": 4,
      "use_weighted_sampler": false,
      "optimizer": {
         "learning_rate": 0.001,
         "eps": 1e-8,
         "weight_decay": 1e-6,
         "betas": [
            0.9,
            0.999
         ],
         "name": "noam",
         "warmup_steps": 1000
      },
      "vocoder_path": "/gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/LJSpeech-1.1/ap-513/logs_and_checkpoints/VocoderExperiment/base/checkpoints/voc.ckpt",
      "mel_loss_weight": 1.0,
      "postnet_loss_weight": 1.0,
      "pitch_loss_weight": 0.1,
      "energy_loss_weight": 0.1,
      "duration_loss_weight": 0.1,
      "attn_ctc_loss_weight": 0.1,
      "attn_bin_loss_weight": 0.1,
      "attn_bin_loss_warmup_epochs": 100
   },
   "path_to_training_config_file": null,
   "preprocessing": {
      "dataset": "ap-513",
      "train_split": 0.9,
      "dataset_split_seed": 1234,
      "save_dir": "/gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/LJSpeech-1.1/ap-513/preprocessed",
      "audio": {
         "min_audio_length": 0.4,
         "max_audio_length": 11.0,
         "max_wav_value": 32767.0,
         "input_sampling_rate": 22050,
         "output_sampling_rate": 22050,
         "alignment_sampling_rate": 22050,
         "target_bit_depth": 16,
         "n_fft": 1024,
         "fft_window_size": 1024,
         "fft_hop_size": 256,
         "f_min": 0,
         "f_max": 8000,
         "n_mels": 80,
         "spec_type": "mel-librosa",
         "vocoder_segment_size": 8192
      },
      "path_to_audio_config_file": null,
      "source_data": [
         {
            "label": "dataset_0",
            "permissions_obtained": true,
            "data_dir": "/gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/LJSpeech-1.1/wavs",
            "filelist": "/gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/LJSpeech-1.1/ap-513/ap-513-filelist.psv",
            "filelist_loader": "everyvoice.utils.generic_dict_loader",
            "sox_effects": [
               [
                  "channels",
                  "1"
               ],
               [
                  "norm",
                  "-3.0"
               ],
               [
                  "silence",
                  "1",
                  "0.1",
                  "0.1%"
               ],
               [
                  "reverse"
               ],
               [
                  "silence",
                  "1",
                  "0.1",
                  "0.1%"
               ],
               [
                  "reverse"
               ]
            ]
         }
      ]
   },
   "path_to_preprocessing_config_file": "/gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/LJSpeech-1.1/ap-513/config/everyvoice-shared-data.yaml",
   "text": {
      "symbols": {
         "silence": [
            "<SIL>"
         ],
         "punctuation": {
            "exclamations": [
               "!",
               "¡"
            ],
            "question_symbols": [
               "?",
               "¿"
            ],
            "quotemarks": [
               "\"",
               "'",
               "“",
               "”",
               "«",
               "»"
            ],
            "big_breaks": [
               ".",
               ":",
               ";"
            ],
            "small_breaks": [
               ",",
               "-",
               "—"
            ],
            "ellipsis": [
               "…"
            ]
         },
         "ap-513_characters": [
            "(",
            ")",
            "a",
            "b",
            "c",
            "d",
            "e",
            "f",
            "g",
            "h",
            "i",
            "j",
            "k",
            "l",
            "m",
            "n",
            "o",
            "p",
            "q",
            "r",
            "s",
            "t",
            "u",
            "v",
            "w",
            "x",
            "y",
            "z",
            "â",
            "é"
         ],
         "ap-513_phones": [
            "a",
            "b",
            "d",
            "e",
            "f",
            "h",
            "i",
            "j",
            "k",
            "l",
            "m",
            "n",
            "o",
            "p",
            "s",
            "t",
            "u",
            "v",
            "w",
            "z",
            "æ",
            "ð",
            "ŋ",
            "ɑ",
            "ɔ",
            "ɛ",
            "ɜ˞",
            "ɡ",
            "ɪ",
            "ɹ",
            "ʃ",
            "ʊ",
            "ʌ",
            "ʒ",
            "θ"
         ]
      },
      "to_replace": {},
      "cleaners": [
         "everyvoice.utils.collapse_whitespace",
         "everyvoice.utils.strip_text",
         "everyvoice.utils.lower",
         "everyvoice.utils.nfc_normalize"
      ]
   },
   "path_to_text_config_file": "/gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/LJSpeech-1.1/ap-513/config/everyvoice-shared-text.yaml"
}
2024-10-30 13:45:29.764 | INFO     | everyvoice.base_cli.helpers:train_base_command:210 - Loading modules for training...

  0%|          | 0/4 [00:00<?, ?it/s]
Loading pytorch and friends:   0%|          | 0/4 [00:00<?, ?it/s]
Loading pytorch and friends:  25%|██▌       | 1/4 [00:00<00:00, 20460.02it/s]
Loading pytorch and friends:  50%|█████     | 2/4 [00:00<00:00, 19152.07it/s]
Loading pytorch and friends:  75%|███████▌  | 3/4 [00:00<00:00, 20295.02it/s]
Loading EveryVoice modules:  75%|███████▌  | 3/4 [00:00<00:00, 16008.79it/s] 
Loading EveryVoice modules: 100%|██████████| 4/4 [00:00<00:00, 17716.17it/s]2024-10-30 13:45:29.768 | INFO     | everyvoice.base_cli.helpers:train_base_command:237 - Starting training.
GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
HPU available: False, using: 0 HPUs
2024-10-30 13:45:31.099 | INFO     | everyvoice.base_cli.helpers:train_base_command:287 - Model's architecture
FastSpeech2(
  (loss): FastSpeech2Loss(
    (attn_ctc_loss): AttentionCTCLoss(
      (log_softmax): LogSoftmax(dim=-1)
      (CTCLoss): CTCLoss()
    )
    (attn_bin_loss): AttentionBinarizationLoss()
  )
  (text_input_layer): Embedding(71, 256, padding_idx=0)
  (position_embedding): PositionalEmbedding()
  (encoder): Conformer(
    (conformer_layers): ModuleList(
      (0-3): 4 x ConformerLayer(
        (ffn1): _FeedForwardModule(
          (sequential): Sequential(
            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
            (1): Linear(in_features=256, out_features=1024, bias=True)
            (2): SiLU()
            (3): Dropout(p=0.2, inplace=False)
            (4): Linear(in_features=1024, out_features=256, bias=True)
            (5): Dropout(p=0.2, inplace=False)
          )
        )
        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
        (self_attn): MultiheadAttention(
          (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True)
        )
        (self_attn_dropout): Dropout(p=0.2, inplace=False)
        (conv_module): _ConvolutionModule(
          (layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
          (sequential): Sequential(
            (0): Conv1d(256, 512, kernel_size=(1,), stride=(1,))
            (1): GLU(dim=1)
            (2): Conv1d(256, 256, kernel_size=(9,), stride=(1,), padding=(4,), groups=256)
            (3): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (4): SiLU()
            (5): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
            (6): Dropout(p=0.2, inplace=False)
          )
        )
        (ffn2): _FeedForwardModule(
          (sequential): Sequential(
            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
            (1): Linear(in_features=256, out_features=1024, bias=True)
            (2): SiLU()
            (3): Dropout(p=0.2, inplace=False)
            (4): Linear(in_features=1024, out_features=256, bias=True)
            (5): Dropout(p=0.2, inplace=False)
          )
        )
        (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
      )
    )
  )
  (variance_adaptor): VarianceAdaptor(
    (duration_predictor): VariancePredictor(
      (conv): ModuleList(
        (0-4): 5 x VarianceConvolutionLayer(
          (layers): Sequential(
            (0): Transpose(
              (module): DepthwiseSeparableConv1d(
                (model): Sequential(
                  (0): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,), groups=256)
                  (1): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
                )
              )
            )
            (1): ReLU()
            (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
            (3): Dropout(p=0.5, inplace=False)
          )
        )
      )
      (linear): Linear(in_features=256, out_features=1, bias=True)
    )
    (length_regulator): LengthRegulator()
    (pitch_predictor): VariancePredictor(
      (conv): ModuleList(
        (0-4): 5 x VarianceConvolutionLayer(
          (layers): Sequential(
            (0): Transpose(
              (module): DepthwiseSeparableConv1d(
                (model): Sequential(
                  (0): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,), groups=256)
                  (1): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
                )
              )
            )
            (1): ReLU()
            (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
            (3): Dropout(p=0.5, inplace=False)
          )
        )
      )
      (linear): Linear(in_features=256, out_features=1, bias=True)
    )
    (pitch_embedding): Embedding(256, 256)
    (energy_predictor): VariancePredictor(
      (conv): ModuleList(
        (0-4): 5 x VarianceConvolutionLayer(
          (layers): Sequential(
            (0): Transpose(
              (module): DepthwiseSeparableConv1d(
                (model): Sequential(
                  (0): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,), groups=256)
                  (1): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
                )
              )
            )
            (1): ReLU()
            (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
            (3): Dropout(p=0.5, inplace=False)
          )
        )
      )
      (linear): Linear(in_features=256, out_features=1, bias=True)
    )
    (energy_embedding): Embedding(256, 256)
    (attention): ConvAttention(
      (softmax): Softmax(dim=3)
      (log_softmax): LogSoftmax(dim=3)
      (query_proj): Sequential(
        (0): ConvNorm(
          (conv): Conv1d(80, 160, kernel_size=(3,), stride=(1,), padding=(1,))
        )
        (1): ReLU()
        (2): ConvNorm(
          (conv): Conv1d(160, 80, kernel_size=(1,), stride=(1,))
        )
        (3): ReLU()
        (4): ConvNorm(
          (conv): Conv1d(80, 80, kernel_size=(1,), stride=(1,))
        )
      )
      (key_proj): Sequential(
        (0): ConvNorm(
          (conv): Conv1d(256, 512, kernel_size=(3,), stride=(1,), padding=(1,))
        )
        (1): ReLU()
        (2): ConvNorm(
          (conv): Conv1d(512, 80, kernel_size=(1,), stride=(1,))
        )
      )
    )
  )
  (decoder): Conformer(
    (conformer_layers): ModuleList(
      (0-3): 4 x ConformerLayer(
        (ffn1): _FeedForwardModule(
          (sequential): Sequential(
            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
            (1): Linear(in_features=256, out_features=1024, bias=True)
            (2): SiLU()
            (3): Dropout(p=0.2, inplace=False)
            (4): Linear(in_features=1024, out_features=256, bias=True)
            (5): Dropout(p=0.2, inplace=False)
          )
        )
        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
        (self_attn): MultiheadAttention(
          (out_proj): NonDynamicallyQuantizableLinear(in_features=256, out_features=256, bias=True)
        )
        (self_attn_dropout): Dropout(p=0.2, inplace=False)
        (conv_module): _ConvolutionModule(
          (layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
          (sequential): Sequential(
            (0): Conv1d(256, 512, kernel_size=(1,), stride=(1,))
            (1): GLU(dim=1)
            (2): Conv1d(256, 256, kernel_size=(9,), stride=(1,), padding=(4,), groups=256)
            (3): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (4): SiLU()
            (5): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
            (6): Dropout(p=0.2, inplace=False)
          )
        )
        (ffn2): _FeedForwardModule(
          (sequential): Sequential(
            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
            (1): Linear(in_features=256, out_features=1024, bias=True)
            (2): SiLU()
            (3): Dropout(p=0.2, inplace=False)
            (4): Linear(in_features=1024, out_features=256, bias=True)
            (5): Dropout(p=0.2, inplace=False)
          )
        )
        (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
      )
    )
  )
  (mel_linear): Linear(in_features=256, out_features=80, bias=True)
  (postnet): PostNet(
    (convolutions): ModuleList(
      (0): Sequential(
        (0): ConvNorm(
          (conv): Conv1d(80, 512, kernel_size=(5,), stride=(1,), padding=(2,))
        )
        (1): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (1-3): 3 x Sequential(
        (0): ConvNorm(
          (conv): Conv1d(512, 512, kernel_size=(5,), stride=(1,), padding=(2,))
        )
        (1): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (4): Sequential(
        (0): ConvNorm(
          (conv): Conv1d(512, 80, kernel_size=(5,), stride=(1,), padding=(2,))
        )
        (1): BatchNorm1d(80, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
  )
)
You are using a CUDA device ('NVIDIA A100-SXM4-40GB') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision
Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/1
----------------------------------------------------------------------------------------------------
distributed_backend=nccl
All distributed processes registered. Starting with 1 processes
----------------------------------------------------------------------------------------------------

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

  | Name               | Type                | Params | Mode 
-------------------------------------------------------------------
0 | loss               | FastSpeech2Loss     | 0      | train
1 | text_input_layer   | Embedding           | 18.2 K | train
2 | position_embedding | PositionalEmbedding | 0      | train
3 | encoder            | Conformer           | 6.1 M  | train
4 | variance_adaptor   | VarianceAdaptor     | 1.6 M  | train
5 | decoder            | Conformer           | 6.1 M  | train
6 | mel_linear         | Linear              | 20.6 K | train
7 | postnet            | PostNet             | 4.3 M  | train
-------------------------------------------------------------------
18.2 M    Trainable params
510       Non-trainable params
18.2 M    Total params
72.641    Total estimated model params size (MB)
471       Modules in train mode
0         Modules in eval mode
SLURM auto-requeueing enabled. Setting signal handlers.
/gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_513/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:424: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=31` in the `DataLoader` to improve performance.
╭───────────────────── Traceback (most recent call last) ──────────────────────╮
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/EveryVoice_dev.ap_513/e │
│ veryvoice/model/feature_prediction/FastSpeech2_lightning/fs2/cli/train.py:33 │
│  in train                                                                    │
│                                                                              │
│   30 │                                                                       │
│   31 │   model_kwargs = {"lang2id": lang2id, "speaker2id": speaker2id, "stat │
│   32 │                                                                       │
│ ❱ 33 │   train_base_command(                                                 │
│   34 │   │   model_config=FastSpeech2Config,                                 │
│   35 │   │   model=FastSpeech2,                                              │
│   36 │   │   data_module=FastSpeech2DataModule,                              │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/EveryVoice_dev.ap_513/e │
│ veryvoice/base_cli/helpers.py:289 in train_base_command                      │
│                                                                              │
│   286 │   │   model_obj = model(config, **model_kwargs)                      │
│   287 │   │   logger.info(f"Model's architecture\n{model_obj}")              │
│   288 │   │   tensorboard_logger.log_hyperparams(config.model_dump())        │
│ ❱ 289 │   │   trainer.fit(model_obj, data)                                   │
│   290 │   else:                                                              │
│   291 │   │   try:                                                           │
│   292 │   │   │   model_obj = model.load_from_checkpoint(last_ckpt)          │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py:538 in │
│ fit                                                                          │
│                                                                              │
│    535 │   │   self.state.fn = TrainerFn.FITTING                             │
│    536 │   │   self.state.status = TrainerStatus.RUNNING                     │
│    537 │   │   self.training = True                                          │
│ ❱  538 │   │   call._call_and_handle_interrupt(                              │
│    539 │   │   │   self, self._fit_impl, model, train_dataloaders, val_datal │
│    540 │   │   )                                                             │
│    541                                                                       │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py:46 in     │
│ _call_and_handle_interrupt                                                   │
│                                                                              │
│    43 │   """                                                                │
│    44 │   try:                                                               │
│    45 │   │   if trainer.strategy.launcher is not None:                      │
│ ❱  46 │   │   │   return trainer.strategy.launcher.launch(trainer_fn, *args, │
│    47 │   │   return trainer_fn(*args, **kwargs)                             │
│    48 │                                                                      │
│    49 │   except _TunerExitException:                                        │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subp │
│ rocess_script.py:105 in launch                                               │
│                                                                              │
│   102 │   │   │   _launch_process_observer(self.procs)                       │
│   103 │   │                                                                  │
│   104 │   │   _set_num_threads_if_needed(num_processes=self.num_processes)   │
│ ❱ 105 │   │   return function(*args, **kwargs)                               │
│   106 │                                                                      │
│   107 │   @override                                                          │
│   108 │   def kill(self, signum: _SIGNUM) -> None:                           │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py:574 in │
│ _fit_impl                                                                    │
│                                                                              │
│    571 │   │   │   model_provided=True,                                      │
│    572 │   │   │   model_connected=self.lightning_module is not None,        │
│    573 │   │   )                                                             │
│ ❱  574 │   │   self._run(model, ckpt_path=ckpt_path)                         │
│    575 │   │                                                                 │
│    576 │   │   assert self.state.stopped                                     │
│    577 │   │   self.training = False                                         │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py:981 in │
│ _run                                                                         │
│                                                                              │
│    978 │   │   # ----------------------------                                │
│    979 │   │   # RUN THE TRAINER                                             │
│    980 │   │   # ----------------------------                                │
│ ❱  981 │   │   results = self._run_stage()                                   │
│    982 │   │                                                                 │
│    983 │   │   # ----------------------------                                │
│    984 │   │   # POST-Training CLEAN UP                                      │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py:1023   │
│ in _run_stage                                                                │
│                                                                              │
│   1020 │   │   │   return self.predict_loop.run()                            │
│   1021 │   │   if self.training:                                             │
│   1022 │   │   │   with isolate_rng():                                       │
│ ❱ 1023 │   │   │   │   self._run_sanity_check()                              │
│   1024 │   │   │   with torch.autograd.set_detect_anomaly(self._detect_anoma │
│   1025 │   │   │   │   self.fit_loop.run()                                   │
│   1026 │   │   │   return None                                               │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py:1052   │
│ in _run_sanity_check                                                         │
│                                                                              │
│   1049 │   │   │   call._call_callback_hooks(self, "on_sanity_check_start")  │
│   1050 │   │   │                                                             │
│   1051 │   │   │   # run eval step                                           │
│ ❱ 1052 │   │   │   val_loop.run()                                            │
│   1053 │   │   │                                                             │
│   1054 │   │   │   call._call_callback_hooks(self, "on_sanity_check_end")    │
│   1055                                                                       │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/pytorch_lightning/loops/utilities.py:178 in │
│ _decorator                                                                   │
│                                                                              │
│   175 │   │   else:                                                          │
│   176 │   │   │   context_manager = torch.no_grad                            │
│   177 │   │   with context_manager():                                        │
│ ❱ 178 │   │   │   return loop_run(self, *args, **kwargs)                     │
│   179 │                                                                      │
│   180 │   return _decorator                                                  │
│   181                                                                        │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py: │
│ 135 in run                                                                   │
│                                                                              │
│   132 │   │   │   │   previous_dataloader_idx = dataloader_idx               │
│   133 │   │   │   │   self.batch_progress.is_last_batch = data_fetcher.done  │
│   134 │   │   │   │   # run step hooks                                       │
│ ❱ 135 │   │   │   │   self._evaluation_step(batch, batch_idx, dataloader_idx │
│   136 │   │   │   except StopIteration:                                      │
│   137 │   │   │   │   # this needs to wrap the `*_step` call too (not just ` │
│   138 │   │   │   │   break                                                  │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py: │
│ 396 in _evaluation_step                                                      │
│                                                                              │
│   393 │   │   │   if not using_dataloader_iter                               │
│   394 │   │   │   else (dataloader_iter,)                                    │
│   395 │   │   )                                                              │
│ ❱ 396 │   │   output = call._call_strategy_hook(trainer, hook_name, *step_ar │
│   397 │   │                                                                  │
│   398 │   │   self.batch_progress.increment_processed()                      │
│   399                                                                        │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py:319 in    │
│ _call_strategy_hook                                                          │
│                                                                              │
│   316 │   │   return None                                                    │
│   317 │                                                                      │
│   318 │   with trainer.profiler.profile(f"[Strategy]{trainer.strategy.__clas │
│ ❱ 319 │   │   output = fn(*args, **kwargs)                                   │
│   320 │                                                                      │
│   321 │   # restore current_fx when nested context                           │
│   322 │   pl_module._current_fx_name = prev_fx_name                          │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py:41 │
│ 0 in validation_step                                                         │
│                                                                              │
│   407 │   │   assert self.model is not None                                  │
│   408 │   │   with self.precision_plugin.val_step_context():                 │
│   409 │   │   │   if self.model != self.lightning_module:                    │
│ ❱ 410 │   │   │   │   return self._forward_redirection(self.model, self.ligh │
│   411 │   │   │   return self.lightning_module.validation_step(*args, **kwar │
│   412 │                                                                      │
│   413 │   def test_step(self, *args: Any, **kwargs: Any) -> STEP_OUTPUT:     │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py:64 │
│ 0 in __call__                                                                │
│                                                                              │
│   637 │   │   # Patch the original_module's forward so we can redirect the a │
│   638 │   │   original_module.forward = wrapped_forward  # type: ignore[meth │
│   639 │   │                                                                  │
│ ❱ 640 │   │   wrapper_output = wrapper_module(*args, **kwargs)               │
│   641 │   │   self.on_after_outer_forward(wrapper_module, original_module)   │
│   642 │   │   return wrapper_output                                          │
│   643                                                                        │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/torch/nn/modules/module.py:1518 in          │
│ _wrapped_call_impl                                                           │
│                                                                              │
│   1515 │   │   if self._compiled_call_impl is not None:                      │
│   1516 │   │   │   return self._compiled_call_impl(*args, **kwargs)  # type: │
│   1517 │   │   else:                                                         │
│ ❱ 1518 │   │   │   return self._call_impl(*args, **kwargs)                   │
│   1519 │                                                                     │
│   1520 │   def _call_impl(self, *args, **kwargs):                            │
│   1521 │   │   forward_call = (self._slow_forward if torch._C._get_tracing_s │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/torch/nn/modules/module.py:1527 in          │
│ _call_impl                                                                   │
│                                                                              │
│   1524 │   │   if not (self._backward_hooks or self._backward_pre_hooks or s │
│   1525 │   │   │   │   or _global_backward_pre_hooks or _global_backward_hoo │
│   1526 │   │   │   │   or _global_forward_hooks or _global_forward_pre_hooks │
│ ❱ 1527 │   │   │   return forward_call(*args, **kwargs)                      │
│   1528 │   │                                                                 │
│   1529 │   │   try:                                                          │
│   1530 │   │   │   result = None                                             │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/torch/nn/parallel/distributed.py:1519 in    │
│ forward                                                                      │
│                                                                              │
│   1516 │   │   │   output = (                                                │
│   1517 │   │   │   │   self.module.forward(*inputs, **kwargs)                │
│   1518 │   │   │   │   if self._delay_all_reduce_all_params                  │
│ ❱ 1519 │   │   │   │   else self._run_ddp_forward(*inputs, **kwargs)         │
│   1520 │   │   │   )                                                         │
│   1521 │   │   │   return self._post_forward(output)                         │
│   1522                                                                       │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/torch/nn/parallel/distributed.py:1355 in    │
│ _run_ddp_forward                                                             │
│                                                                              │
│   1352 │                                                                     │
│   1353 │   def _run_ddp_forward(self, *inputs, **kwargs):                    │
│   1354 │   │   with self._inside_ddp_forward():                              │
│ ❱ 1355 │   │   │   return self.module(*inputs, **kwargs)  # type: ignore[ind │
│   1356 │                                                                     │
│   1357 │   def _clear_grad_buffer(self):                                     │
│   1358 │   │   # Making param.grad points to the grad buffers before backwar │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/torch/nn/modules/module.py:1518 in          │
│ _wrapped_call_impl                                                           │
│                                                                              │
│   1515 │   │   if self._compiled_call_impl is not None:                      │
│   1516 │   │   │   return self._compiled_call_impl(*args, **kwargs)  # type: │
│   1517 │   │   else:                                                         │
│ ❱ 1518 │   │   │   return self._call_impl(*args, **kwargs)                   │
│   1519 │                                                                     │
│   1520 │   def _call_impl(self, *args, **kwargs):                            │
│   1521 │   │   forward_call = (self._slow_forward if torch._C._get_tracing_s │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/torch/nn/modules/module.py:1527 in          │
│ _call_impl                                                                   │
│                                                                              │
│   1524 │   │   if not (self._backward_hooks or self._backward_pre_hooks or s │
│   1525 │   │   │   │   or _global_backward_pre_hooks or _global_backward_hoo │
│   1526 │   │   │   │   or _global_forward_hooks or _global_forward_pre_hooks │
│ ❱ 1527 │   │   │   return forward_call(*args, **kwargs)                      │
│   1528 │   │                                                                 │
│   1529 │   │   try:                                                          │
│   1530 │   │   │   result = None                                             │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py:63 │
│ 3 in wrapped_forward                                                         │
│                                                                              │
│   630 │   │   │   original_module.forward = original_forward  # type: ignore │
│   631 │   │   │   # Call the actual method e.g. `.training_step(...)`        │
│   632 │   │   │   method = getattr(original_module, method_name)             │
│ ❱ 633 │   │   │   out = method(*_args, **_kwargs)                            │
│   634 │   │   │   self.on_after_inner_forward(wrapper_module, original_modul │
│   635 │   │   │   return out                                                 │
│   636                                                                        │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/EveryVoice_dev.ap_513/e │
│ veryvoice/model/feature_prediction/FastSpeech2_lightning/fs2/model.py:408 in │
│ validation_step                                                              │
│                                                                              │
│   405 │                                                                      │
│   406 │   def validation_step(self, batch, batch_idx):                       │
│   407 │   │   if self.global_step == 0:                                      │
│ ❱ 408 │   │   │   self._validation_global_step_0(batch, batch_idx)           │
│   409 │   │                                                                  │
│   410 │   │   output = self(batch)                                           │
│   411 │   │   if batch_idx == 0:                                             │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/EveryVoice_dev.ap_513/e │
│ veryvoice/model/feature_prediction/FastSpeech2_lightning/fs2/model.py:323 in │
│ _validation_global_step_0                                                    │
│                                                                              │
│   320 │   │   │   vocoder_model, vocoder_config = load_hifigan_from_checkpoi │
│   321 │   │   │   │   vocoder_ckpt, input_.device                            │
│   322 │   │   │   )                                                          │
│ ❱ 323 │   │   │   wav, sr = synthesize_data(input_, vocoder_model, vocoder_c │
│   324 │   │   │                                                              │
│   325 │   │   │   self.logger.experiment.add_audio(                          │
│   326 │   │   │   │   f"copy-synthesis/wav_{batch['basename'][0]}",          │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/EveryVoice_dev.ap_513/e │
│ veryvoice/model/vocoder/HiFiGAN_iSTFT_lightning/hfgl/utils.py:82 in          │
│ synthesize_data                                                              │
│                                                                              │
│    79 │   │   wavs = inverse_spectral_transform(mag * torch.exp(phase * 1j)) │
│    80 │   else:                                                              │
│    81 │   │   with torch.no_grad():                                          │
│ ❱  82 │   │   │   wavs = model.generator(data)                               │
│    83 │   return (                                                           │
│    84 │   │   wavs.cpu(),                                                    │
│    85 │   │   config.preprocessing.audio.output_sampling_rate,               │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/torch/nn/modules/module.py:1518 in          │
│ _wrapped_call_impl                                                           │
│                                                                              │
│   1515 │   │   if self._compiled_call_impl is not None:                      │
│   1516 │   │   │   return self._compiled_call_impl(*args, **kwargs)  # type: │
│   1517 │   │   else:                                                         │
│ ❱ 1518 │   │   │   return self._call_impl(*args, **kwargs)                   │
│   1519 │                                                                     │
│   1520 │   def _call_impl(self, *args, **kwargs):                            │
│   1521 │   │   forward_call = (self._slow_forward if torch._C._get_tracing_s │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/torch/nn/modules/module.py:1527 in          │
│ _call_impl                                                                   │
│                                                                              │
│   1524 │   │   if not (self._backward_hooks or self._backward_pre_hooks or s │
│   1525 │   │   │   │   or _global_backward_pre_hooks or _global_backward_hoo │
│   1526 │   │   │   │   or _global_forward_hooks or _global_forward_pre_hooks │
│ ❱ 1527 │   │   │   return forward_call(*args, **kwargs)                      │
│   1528 │   │                                                                 │
│   1529 │   │   try:                                                          │
│   1530 │   │   │   result = None                                             │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/EveryVoice_dev.ap_513/e │
│ veryvoice/model/vocoder/HiFiGAN_iSTFT_lightning/hfgl/model.py:230 in forward │
│                                                                              │
│   227 │   │   self.conv_post.apply(init_weights)                             │
│   228 │                                                                      │
│   229 │   def forward(self, x):                                              │
│ ❱ 230 │   │   x = self.conv_pre(x)                                           │
│   231 │   │   for i in range(self.num_upsamples):                            │
│   232 │   │   │   x = self.config.model.activation_function(x)               │
│   233 │   │   │   x = self.ups[i](x)                                         │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/torch/nn/modules/module.py:1518 in          │
│ _wrapped_call_impl                                                           │
│                                                                              │
│   1515 │   │   if self._compiled_call_impl is not None:                      │
│   1516 │   │   │   return self._compiled_call_impl(*args, **kwargs)  # type: │
│   1517 │   │   else:                                                         │
│ ❱ 1518 │   │   │   return self._call_impl(*args, **kwargs)                   │
│   1519 │                                                                     │
│   1520 │   def _call_impl(self, *args, **kwargs):                            │
│   1521 │   │   forward_call = (self._slow_forward if torch._C._get_tracing_s │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/torch/nn/modules/module.py:1527 in          │
│ _call_impl                                                                   │
│                                                                              │
│   1524 │   │   if not (self._backward_hooks or self._backward_pre_hooks or s │
│   1525 │   │   │   │   or _global_backward_pre_hooks or _global_backward_hoo │
│   1526 │   │   │   │   or _global_forward_hooks or _global_forward_pre_hooks │
│ ❱ 1527 │   │   │   return forward_call(*args, **kwargs)                      │
│   1528 │   │                                                                 │
│   1529 │   │   try:                                                          │
│   1530 │   │   │   result = None                                             │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/torch/nn/modules/conv.py:310 in forward     │
│                                                                              │
│    307 │   │   │   │   │   │   self.padding, self.dilation, self.groups)     │
│    308 │                                                                     │
│    309 │   def forward(self, input: Tensor) -> Tensor:                       │
│ ❱  310 │   │   return self._conv_forward(input, self.weight, self.bias)      │
│    311                                                                       │
│    312                                                                       │
│    313 class Conv2d(_ConvNd):                                                │
│                                                                              │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_dev.ap_ │
│ 513/lib/python3.10/site-packages/torch/nn/modules/conv.py:306 in             │
│ _conv_forward                                                                │
│                                                                              │
│    303 │   │   │   return F.conv1d(F.pad(input, self._reversed_padding_repea │
│    304 │   │   │   │   │   │   │   weight, bias, self.stride,                │
│    305 │   │   │   │   │   │   │   _single(0), self.dilation, self.groups)   │
│ ❱  306 │   │   return F.conv1d(input, weight, bias, self.stride,             │
│    307 │   │   │   │   │   │   self.padding, self.dilation, self.groups)     │
│    308 │                                                                     │
│    309 │   def forward(self, input: Tensor) -> Tensor:                       │
╰──────────────────────────────────────────────────────────────────────────────╯
RuntimeError: Given groups=1, weight of size [512, 80, 7], expected input[1, 
696, 80] to have 80 channels, but got 696 channels instead

Loading EveryVoice modules: 100%|██████████| 4/4 [00:13<00:00,  3.35s/it]   
srun: error: ib14gpu-002: task 0: Exited with exit code 1