From e01170091a81ec1528057df2a437ffac1a335dec Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-22-95.ec2.internal>
Date: Tue, 22 Mar 2022 03:18:20 +0000
Subject: [PATCH 01/56] Add MediaSpeech ASR recipe

---
 egs2/README.md                                |   1 +
 egs2/TEMPLATE/asr1/db.sh                      |   1 +
 egs2/mediaspeech/asr1/README.md               |   0
 egs2/mediaspeech/asr1/asr.sh                  |   1 +
 egs2/mediaspeech/asr1/cmd.sh                  | 110 ++++++++++++++++++
 egs2/mediaspeech/asr1/conf/decode_asr.yaml    |   1 +
 egs2/mediaspeech/asr1/conf/fbank.conf         |   2 +
 egs2/mediaspeech/asr1/conf/pbs.conf           |  11 ++
 egs2/mediaspeech/asr1/conf/pitch.conf         |   1 +
 egs2/mediaspeech/asr1/conf/queue.conf         |  12 ++
 egs2/mediaspeech/asr1/conf/slurm.conf         |  14 +++
 egs2/mediaspeech/asr1/conf/train_asr.yaml     |   1 +
 .../asr1/conf/tuning/train_asr_conformer.yaml |  70 +++++++++++
 .../asr1/conf/tuning/train_asr_fused.yaml     |  96 +++++++++++++++
 .../asr1/conf/tuning/train_asr_hubert.yaml    |  86 ++++++++++++++
 .../conf/tuning/train_asr_transformer.yaml    |  48 ++++++++
 egs2/mediaspeech/asr1/db.sh                   |   1 +
 egs2/mediaspeech/asr1/local/data.sh           |  82 +++++++++++++
 egs2/mediaspeech/asr1/local/data_prep.pl      |   1 +
 egs2/mediaspeech/asr1/local/data_prep.py      |  92 +++++++++++++++
 .../asr1/local/download_and_untar.sh          |   1 +
 egs2/mediaspeech/asr1/local/path.sh           |   0
 egs2/mediaspeech/asr1/path.sh                 |   1 +
 egs2/mediaspeech/asr1/pyscripts               |   1 +
 egs2/mediaspeech/asr1/run.sh                  |  45 +++++++
 egs2/mediaspeech/asr1/scripts                 |   1 +
 egs2/mediaspeech/asr1/steps                   |   1 +
 egs2/mediaspeech/asr1/utils                   |   1 +
 28 files changed, 682 insertions(+)
 create mode 100644 egs2/mediaspeech/asr1/README.md
 create mode 120000 egs2/mediaspeech/asr1/asr.sh
 create mode 100644 egs2/mediaspeech/asr1/cmd.sh
 create mode 120000 egs2/mediaspeech/asr1/conf/decode_asr.yaml
 create mode 100644 egs2/mediaspeech/asr1/conf/fbank.conf
 create mode 100644 egs2/mediaspeech/asr1/conf/pbs.conf
 create mode 100644 egs2/mediaspeech/asr1/conf/pitch.conf
 create mode 100644 egs2/mediaspeech/asr1/conf/queue.conf
 create mode 100644 egs2/mediaspeech/asr1/conf/slurm.conf
 create mode 120000 egs2/mediaspeech/asr1/conf/train_asr.yaml
 create mode 100644 egs2/mediaspeech/asr1/conf/tuning/train_asr_conformer.yaml
 create mode 100644 egs2/mediaspeech/asr1/conf/tuning/train_asr_fused.yaml
 create mode 100644 egs2/mediaspeech/asr1/conf/tuning/train_asr_hubert.yaml
 create mode 100644 egs2/mediaspeech/asr1/conf/tuning/train_asr_transformer.yaml
 create mode 120000 egs2/mediaspeech/asr1/db.sh
 create mode 100755 egs2/mediaspeech/asr1/local/data.sh
 create mode 120000 egs2/mediaspeech/asr1/local/data_prep.pl
 create mode 100755 egs2/mediaspeech/asr1/local/data_prep.py
 create mode 120000 egs2/mediaspeech/asr1/local/download_and_untar.sh
 create mode 100644 egs2/mediaspeech/asr1/local/path.sh
 create mode 120000 egs2/mediaspeech/asr1/path.sh
 create mode 120000 egs2/mediaspeech/asr1/pyscripts
 create mode 100755 egs2/mediaspeech/asr1/run.sh
 create mode 120000 egs2/mediaspeech/asr1/scripts
 create mode 120000 egs2/mediaspeech/asr1/steps
 create mode 120000 egs2/mediaspeech/asr1/utils

diff --git a/egs2/README.md b/egs2/README.md
index 133fc9192f6..5aae03496fb 100755
--- a/egs2/README.md
+++ b/egs2/README.md
@@ -54,6 +54,7 @@ See: https://espnet.github.io/espnet/espnet2_tutorial.html#recipes-using-espnet2
 | ljspeech                | The LJ Speech Dataset                                                                   | TTS                     | ENG                  | https://keithito.com/LJ-Speech-Dataset/                                                                      |              |
 | lrs3                    | The Oxford-BBC Lip Reading Sentences 3 (LRS3) Dataset                                   | ASR                     | ENG                  | https://www.robots.ox.ac.uk/~vgg/data/lip_reading/lrs3.html                                                  |              |
 | lrs2                    | The Oxford-BBC Lip Reading Sentences 2 (LRS2) Dataset                                   | Lipreading/ASR          | ENG                  | https://www.robots.ox.ac.uk/~vgg/data/lip_reading/lrs2.html                                                  |              |
+| mediaspeech             | MediaSpeech: Multilanguage ASR Benchmark and Dataset                                    | ASR                     | FR                   | https://www.openslr.org/108/                                                                           |              |
 | mini_an4                | Mini version of CMU AN4 database for the integration test                               | ASR/TTS/SE              | ENG                  | http://www.speech.cs.cmu.edu/databases/an4/                                                                  |              |
 | mini_librispeech        | Mini version of Librispeech corpus                                                      | DIAR                    | ENG                  | https://openslr.org/31/                                                                                      |              |
 | mls                     | MLS (A large multilingual corpus derived from LibriVox audiobooks)                      | ASR                     | 8 languages          | http://www.openslr.org/94/                                                                                   |              |
diff --git a/egs2/TEMPLATE/asr1/db.sh b/egs2/TEMPLATE/asr1/db.sh
index f7d686fa164..48f2d29a29f 100755
--- a/egs2/TEMPLATE/asr1/db.sh
+++ b/egs2/TEMPLATE/asr1/db.sh
@@ -42,6 +42,7 @@ LIBRILIGHT_LIMITED=
 FSC=
 SLURP=
 VOXCELEB=
+MEDIASPEECH=downloads
 MINI_LIBRISPEECH=downloads
 MISP2021=
 LIBRIMIX=downloads
diff --git a/egs2/mediaspeech/asr1/README.md b/egs2/mediaspeech/asr1/README.md
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/egs2/mediaspeech/asr1/asr.sh b/egs2/mediaspeech/asr1/asr.sh
new file mode 120000
index 00000000000..60b05122cfd
--- /dev/null
+++ b/egs2/mediaspeech/asr1/asr.sh
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/asr.sh
\ No newline at end of file
diff --git a/egs2/mediaspeech/asr1/cmd.sh b/egs2/mediaspeech/asr1/cmd.sh
new file mode 100644
index 00000000000..2aae6919fef
--- /dev/null
+++ b/egs2/mediaspeech/asr1/cmd.sh
@@ -0,0 +1,110 @@
+# ====== About run.pl, queue.pl, slurm.pl, and ssh.pl ======
+# Usage: <cmd>.pl [options] JOB=1:<nj> <log> <command...>
+# e.g.
+#   run.pl --mem 4G JOB=1:10 echo.JOB.log echo JOB
+#
+# Options:
+#   --time <time>: Limit the maximum time to execute.
+#   --mem <mem>: Limit the maximum memory usage.
+#   -–max-jobs-run <njob>: Limit the number parallel jobs. This is ignored for non-array jobs.
+#   --num-threads <ngpu>: Specify the number of CPU core.
+#   --gpu <ngpu>: Specify the number of GPU devices.
+#   --config: Change the configuration file from default.
+#
+# "JOB=1:10" is used for "array jobs" and it can control the number of parallel jobs.
+# The left string of "=", i.e. "JOB", is replaced by <N>(Nth job) in the command and the log file name,
+# e.g. "echo JOB" is changed to "echo 3" for the 3rd job and "echo 8" for 8th job respectively.
+# Note that the number must start with a positive number, so you can't use "JOB=0:10" for example.
+#
+# run.pl, queue.pl, slurm.pl, and ssh.pl have unified interface, not depending on its backend.
+# These options are mapping to specific options for each backend and
+# it is configured by "conf/queue.conf" and "conf/slurm.conf" by default.
+# If jobs failed, your configuration might be wrong for your environment.
+#
+#
+# The official documentation for run.pl, queue.pl, slurm.pl, and ssh.pl:
+#   "Parallelization in Kaldi": http://kaldi-asr.org/doc/queue.html
+# =========================================================~
+
+
+# Select the backend used by run.sh from "local", "stdout", "sge", "slurm", or "ssh"
+cmd_backend='local'
+
+# Local machine, without any Job scheduling system
+if [ "${cmd_backend}" = local ]; then
+
+    # The other usage
+    export train_cmd="run.pl"
+    # Used for "*_train.py": "--gpu" is appended optionally by run.sh
+    export cuda_cmd="run.pl"
+    # Used for "*_recog.py"
+    export decode_cmd="run.pl"
+
+# Local machine logging to stdout and log file, without any Job scheduling system
+elif [ "${cmd_backend}" = stdout ]; then
+
+    # The other usage
+    export train_cmd="stdout.pl"
+    # Used for "*_train.py": "--gpu" is appended optionally by run.sh
+    export cuda_cmd="stdout.pl"
+    # Used for "*_recog.py"
+    export decode_cmd="stdout.pl"
+
+
+# "qsub" (Sun Grid Engine, or derivation of it)
+elif [ "${cmd_backend}" = sge ]; then
+    # The default setting is written in conf/queue.conf.
+    # You must change "-q g.q" for the "queue" for your environment.
+    # To know the "queue" names, type "qhost -q"
+    # Note that to use "--gpu *", you have to setup "complex_value" for the system scheduler.
+
+    export train_cmd="queue.pl"
+    export cuda_cmd="queue.pl"
+    export decode_cmd="queue.pl"
+
+
+# "qsub" (Torque/PBS.)
+elif [ "${cmd_backend}" = pbs ]; then
+    # The default setting is written in conf/pbs.conf.
+
+    export train_cmd="pbs.pl"
+    export cuda_cmd="pbs.pl"
+    export decode_cmd="pbs.pl"
+
+
+# "sbatch" (Slurm)
+elif [ "${cmd_backend}" = slurm ]; then
+    # The default setting is written in conf/slurm.conf.
+    # You must change "-p cpu" and "-p gpu" for the "partition" for your environment.
+    # To know the "partion" names, type "sinfo".
+    # You can use "--gpu * " by default for slurm and it is interpreted as "--gres gpu:*"
+    # The devices are allocated exclusively using "${CUDA_VISIBLE_DEVICES}".
+
+    export train_cmd="slurm.pl"
+    export cuda_cmd="slurm.pl"
+    export decode_cmd="slurm.pl"
+
+elif [ "${cmd_backend}" = ssh ]; then
+    # You have to create ".queue/machines" to specify the host to execute jobs.
+    # e.g. .queue/machines
+    #   host1
+    #   host2
+    #   host3
+    # Assuming you can login them without any password, i.e. You have to set ssh keys.
+
+    export train_cmd="ssh.pl"
+    export cuda_cmd="ssh.pl"
+    export decode_cmd="ssh.pl"
+
+# This is an example of specifying several unique options in the JHU CLSP cluster setup.
+# Users can modify/add their own command options according to their cluster environments.
+elif [ "${cmd_backend}" = jhu ]; then
+
+    export train_cmd="queue.pl --mem 2G"
+    export cuda_cmd="queue-freegpu.pl --mem 2G --gpu 1 --config conf/queue.conf"
+    export decode_cmd="queue.pl --mem 4G"
+
+else
+    echo "$0: Error: Unknown cmd_backend=${cmd_backend}" 1>&2
+    return 1
+fi
diff --git a/egs2/mediaspeech/asr1/conf/decode_asr.yaml b/egs2/mediaspeech/asr1/conf/decode_asr.yaml
new file mode 120000
index 00000000000..f3f59d5ac2b
--- /dev/null
+++ b/egs2/mediaspeech/asr1/conf/decode_asr.yaml
@@ -0,0 +1 @@
+tuning/decode_transformer.yaml
\ No newline at end of file
diff --git a/egs2/mediaspeech/asr1/conf/fbank.conf b/egs2/mediaspeech/asr1/conf/fbank.conf
new file mode 100644
index 00000000000..82ac7bd0dbc
--- /dev/null
+++ b/egs2/mediaspeech/asr1/conf/fbank.conf
@@ -0,0 +1,2 @@
+--sample-frequency=16000 
+--num-mel-bins=80
diff --git a/egs2/mediaspeech/asr1/conf/pbs.conf b/egs2/mediaspeech/asr1/conf/pbs.conf
new file mode 100644
index 00000000000..119509938ce
--- /dev/null
+++ b/egs2/mediaspeech/asr1/conf/pbs.conf
@@ -0,0 +1,11 @@
+# Default configuration
+command qsub -V -v PATH -S /bin/bash
+option name=* -N $0
+option mem=* -l mem=$0
+option mem=0          # Do not add anything to qsub_opts
+option num_threads=* -l ncpus=$0
+option num_threads=1  # Do not add anything to qsub_opts
+option num_nodes=* -l nodes=$0:ppn=1
+default gpu=0
+option gpu=0
+option gpu=* -l ngpus=$0
diff --git a/egs2/mediaspeech/asr1/conf/pitch.conf b/egs2/mediaspeech/asr1/conf/pitch.conf
new file mode 100644
index 00000000000..e959a19d5b8
--- /dev/null
+++ b/egs2/mediaspeech/asr1/conf/pitch.conf
@@ -0,0 +1 @@
+--sample-frequency=16000
diff --git a/egs2/mediaspeech/asr1/conf/queue.conf b/egs2/mediaspeech/asr1/conf/queue.conf
new file mode 100644
index 00000000000..500582fab31
--- /dev/null
+++ b/egs2/mediaspeech/asr1/conf/queue.conf
@@ -0,0 +1,12 @@
+# Default configuration
+command qsub -v PATH -cwd -S /bin/bash -j y -l arch=*64*
+option name=* -N $0
+option mem=* -l mem_free=$0,ram_free=$0
+option mem=0          # Do not add anything to qsub_opts
+option num_threads=* -pe smp $0
+option num_threads=1  # Do not add anything to qsub_opts
+option max_jobs_run=* -tc $0
+option num_nodes=* -pe mpi $0  # You must set this PE as allocation_rule=1
+default gpu=0
+option gpu=0
+option gpu=* -l gpu=$0 -q g.q
diff --git a/egs2/mediaspeech/asr1/conf/slurm.conf b/egs2/mediaspeech/asr1/conf/slurm.conf
new file mode 100644
index 00000000000..3b229673638
--- /dev/null
+++ b/egs2/mediaspeech/asr1/conf/slurm.conf
@@ -0,0 +1,14 @@
+# Default configuration
+command sbatch --export=PATH
+option name=* --job-name $0
+option time=* --time $0
+option mem=* --mem-per-cpu $0
+option mem=0
+option num_threads=* --cpus-per-task $0
+option num_threads=1 --cpus-per-task 1
+option num_nodes=* --nodes $0
+default gpu=0
+option gpu=0 -p cpu
+option gpu=* -p gpu --gres=gpu:$0 -c $0  # Recommend allocating more CPU than, or equal to the number of GPU
+# note: the --max-jobs-run option is supported as a special case
+# by slurm.pl and you don't have to handle it in the config file.
diff --git a/egs2/mediaspeech/asr1/conf/train_asr.yaml b/egs2/mediaspeech/asr1/conf/train_asr.yaml
new file mode 120000
index 00000000000..00ea2f24791
--- /dev/null
+++ b/egs2/mediaspeech/asr1/conf/train_asr.yaml
@@ -0,0 +1 @@
+tuning/train_asr_transformer.yaml
\ No newline at end of file
diff --git a/egs2/mediaspeech/asr1/conf/tuning/train_asr_conformer.yaml b/egs2/mediaspeech/asr1/conf/tuning/train_asr_conformer.yaml
new file mode 100644
index 00000000000..8ad9f0d693f
--- /dev/null
+++ b/egs2/mediaspeech/asr1/conf/tuning/train_asr_conformer.yaml
@@ -0,0 +1,70 @@
+cudnn_benchmark: false 
+cudnn_deterministic: false 
+use_amp: true 
+encoder: conformer
+encoder_conf:
+    output_size: 512
+    attention_heads: 8
+    linear_units: 2048
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d
+    normalize_before: true
+    macaron_style: true
+    pos_enc_layer_type: "rel_pos"
+    selfattention_layer_type: "rel_selfattn"
+    activation_type: "swish"
+    use_cnn_module:  true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 8
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+optim: adam
+optim_conf:
+    lr: 0.0002
+scheduler: warmuplr     # pytorch v1.1.0+ required #Tune warmup steps
+scheduler_conf:
+    warmup_steps: 25000
+max_epoch: 50
+
+frontend_conf:
+  n_fft: 512
+  hop_length: 256
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: "False" means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend.
+
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 40
+    num_time_mask: 2
+
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10 
\ No newline at end of file
diff --git a/egs2/mediaspeech/asr1/conf/tuning/train_asr_fused.yaml b/egs2/mediaspeech/asr1/conf/tuning/train_asr_fused.yaml
new file mode 100644
index 00000000000..38727e5a2bf
--- /dev/null
+++ b/egs2/mediaspeech/asr1/conf/tuning/train_asr_fused.yaml
@@ -0,0 +1,96 @@
+encoder: transformer
+encoder_conf:
+    input_layer: "conv2d"
+    num_blocks: 12
+    linear_units: 2048
+    dropout_rate: 0.1
+    output_size: 256  # dimension of attention
+    attention_heads: 4
+    attention_dropout_rate: 0.0
+
+decoder: transformer
+decoder_conf:
+    input_layer: "embed"
+    num_blocks: 6
+    linear_units: 2048
+    dropout_rate: 0.1
+
+#frontend related 
+freeze_param: ["frontend.upstream"] 
+
+frontend: fused
+frontend_conf: 
+  frontends:
+    - frontend_type: s3prl
+      frontend_conf:
+        upstream: hubert_large_ll60k
+      download_dir: ./hub
+      multilayer_feature: True
+      
+    - frontend_type: default
+      n_fft: 512
+      win_length: 400
+      hop_length: 160
+
+    - frontend_type: s3prl
+      frontend_conf:
+        upstream: wav2vec2_large_ll60k
+      download_dir: ./hub
+      multilayer_feature: True
+
+  align_method: linear_projection
+  proj_dim: 100     
+
+preencoder: linear
+preencoder_conf:
+    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+# hybrid CTC/attention
+model_conf:
+    ctc_weight: 0.1
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: "False" means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend.
+
+
+# minibatch related
+batch_type: folded
+batch_size: 32
+
+# optimization related
+optim: adam
+accum_grad: 2
+grad_clip: 5
+patience: 15
+max_epoch: 100
+optim_conf:
+    lr: 1
+scheduler: noamlr
+scheduler_conf:
+    warmup_steps: 4000
+
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 40
+    num_time_mask: 2
+
+# others:
+best_model_criterion:
+-  - valid
+   - acc
+   - max
+keep_nbest_models: 5
+
+init: xavier_uniform
diff --git a/egs2/mediaspeech/asr1/conf/tuning/train_asr_hubert.yaml b/egs2/mediaspeech/asr1/conf/tuning/train_asr_hubert.yaml
new file mode 100644
index 00000000000..db22456c95d
--- /dev/null
+++ b/egs2/mediaspeech/asr1/conf/tuning/train_asr_hubert.yaml
@@ -0,0 +1,86 @@
+cudnn_benchmark: false 
+cudnn_deterministic: false 
+use_amp: true 
+encoder: conformer
+encoder_conf:
+    output_size: 512
+    attention_heads: 8
+    linear_units: 2048
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d
+    normalize_before: true
+    macaron_style: true
+    pos_enc_layer_type: "rel_pos"
+    selfattention_layer_type: "rel_selfattn"
+    activation_type: "swish"
+    use_cnn_module:  true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 8
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+optim: adam
+optim_conf:
+    lr: 0.0002
+scheduler: warmuplr     # pytorch v1.1.0+ required #Tune warmup steps
+scheduler_conf:
+    warmup_steps: 25000
+max_epoch: 50
+
+freeze_param: [
+"frontend.upstream"
+]
+
+frontend_conf:
+  n_fft: 512
+  hop_length: 256
+
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream: hubert_large_ll60k  # Note: If the upstream is changed, please change the input_size in the preencoder.
+    download_dir: ./hub
+    multilayer_feature: True
+
+preencoder: linear
+preencoder_conf:
+    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: "False" means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend.
+
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 40
+    num_time_mask: 2
+
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10 
\ No newline at end of file
diff --git a/egs2/mediaspeech/asr1/conf/tuning/train_asr_transformer.yaml b/egs2/mediaspeech/asr1/conf/tuning/train_asr_transformer.yaml
new file mode 100644
index 00000000000..ac6366ffb1e
--- /dev/null
+++ b/egs2/mediaspeech/asr1/conf/tuning/train_asr_transformer.yaml
@@ -0,0 +1,48 @@
+encoder: transformer
+encoder_conf:
+    input_layer: "conv2d"
+    num_blocks: 12
+    linear_units: 2048
+    dropout_rate: 0.1
+    output_size: 256
+    attention_heads: 4
+    attention_dropout_rate: 0.0
+
+decoder: transformer
+decoder_conf:
+    input_layer: "embed"
+    num_blocks: 6
+    linear_units: 2048
+    dropout_rate: 0.1
+
+# hybrid CTC/attention
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+
+# minibatch related
+batch_type: folded
+batch_size: 32
+
+# optimization related
+optim: adam
+accum_grad: 2
+grad_clip: 5
+patience: 0
+max_epoch: 10
+optim_conf:
+    lr: 1.0
+scheduler: noamlr
+scheduler_conf:
+    warmup_steps: 25000
+
+# others:
+best_model_criterion:
+-  - valid
+   - acc
+   - max
+keep_nbest_models: 10
+
+init: xavier_uniform # Our empirical studies shows that this initialization
+                     # is very important to low-resource ASR training
diff --git a/egs2/mediaspeech/asr1/db.sh b/egs2/mediaspeech/asr1/db.sh
new file mode 120000
index 00000000000..50d86130898
--- /dev/null
+++ b/egs2/mediaspeech/asr1/db.sh
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/db.sh
\ No newline at end of file
diff --git a/egs2/mediaspeech/asr1/local/data.sh b/egs2/mediaspeech/asr1/local/data.sh
new file mode 100755
index 00000000000..d5d2aa6d9c9
--- /dev/null
+++ b/egs2/mediaspeech/asr1/local/data.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 Johns Hopkins University (Shinji Watanabe)
+#  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)
+
+. ./path.sh || exit 1;
+. ./cmd.sh || exit 1;
+. ./db.sh || exit 1;
+
+# general configuration
+stage=0       # start from 0 if you need to start from data preparation
+stop_stage=100
+SECONDS=0
+lang=en # en de fr cy tt kab ca zh-TW it fa eu es ru tr nl eo zh-CN rw pt zh-HK cs pl uk 
+
+ . utils/parse_options.sh || exit 1;
+
+# base url for downloads.
+# Deprecated url:https://voice-prod-bundler-ee1969a6ce8178826482b88e843c335139bd3fb4.s3.amazonaws.com/cv-corpus-3/$lang.tar.gz
+data_url=https://us.openslr.org/resources/108/FR.tgz
+
+log() {
+    local fname=${BASH_SOURCE[1]##*/}
+    echo -e "$(date '+%Y-%m-%dT%H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
+}
+
+mkdir -p ${COMMONVOICE}
+if [ -z "${COMMONVOICE}" ]; then
+    log "Fill the value of 'COMMONVOICE' of db.sh"
+    exit 1
+fi
+
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+train_set=train_"$(echo "${lang}" | tr - _)"
+train_dev=dev_"$(echo "${lang}" | tr - _)"
+test_set=test_"$(echo "${lang}" | tr - _)"
+
+log "data preparation started"
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then 
+    log "stage1: Download data to ${COMMONVOICE}"
+    log "The default data of this recipe is from commonvoice 5.1, for newer version, you need to register at \
+         https://commonvoice.mozilla.org/"
+    local/download_and_untar.sh ${COMMONVOICE} ${data_url} FR.tgz
+fi
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    log "stage2: Preparing data for commonvoice"
+    ### Task dependent. You have to make data the following preparation part by yourself.
+    mkdir -p data/{train_as,dev_as,test_as,validated_as}
+    python3 local/data_prep.py \
+        --data_path ${COMMONVOICE}/FR \
+        --train_dir data/train_as \
+        --dev_dir data/dev_as \
+        --test_dir data/test_as \
+        --validated_dir data/validated_as \
+        --dev_ratio 0.1 \
+        --test_ratio 0.1 \
+        --validated_ratio 0.01
+    for x in train_as dev_as test_as validated_as; do
+        utils/utt2spk_to_spk2utt.pl data/${x}/utt2spk > data/${x}/spk2utt
+        utils/fix_data_dir.sh data/${x}
+        utils/validate_data_dir.sh --no-feats data/${x}
+    # for part in "validated" "test" "dev"; do
+    #     # use underscore-separated names in data directories.
+    #     local/data_prep.pl "${COMMONVOICE}/cv-corpus-5.1-2020-06-22/${lang}" ${part} data/"$(echo "${part}_${lang}" | tr - _)"
+
+
+    # remove test&dev data from validated sentences
+    # utils/copy_data_dir.sh data/"$(echo "validated_${lang}" | tr - _)" data/${train_set}
+    # utils/filter_scp.pl --exclude data/${train_dev}/wav.scp data/${train_set}/wav.scp > data/${train_set}/temp_wav.scp
+    # utils/filter_scp.pl --exclude data/${test_set}/wav.scp data/${train_set}/temp_wav.scp > data/${train_set}/wav.scp
+    # utils/fix_data_dir.sh data/${train_set}
+    done
+fi
+
+log "Successfully finished. [elapsed=${SECONDS}s]"
diff --git a/egs2/mediaspeech/asr1/local/data_prep.pl b/egs2/mediaspeech/asr1/local/data_prep.pl
new file mode 120000
index 00000000000..28fe16ef888
--- /dev/null
+++ b/egs2/mediaspeech/asr1/local/data_prep.pl
@@ -0,0 +1 @@
+../../../../egs/commonvoice/asr1/local/data_prep.pl
\ No newline at end of file
diff --git a/egs2/mediaspeech/asr1/local/data_prep.py b/egs2/mediaspeech/asr1/local/data_prep.py
new file mode 100755
index 00000000000..59aacfd172e
--- /dev/null
+++ b/egs2/mediaspeech/asr1/local/data_prep.py
@@ -0,0 +1,92 @@
+import os
+import os.path
+import json
+import glob
+import math
+import argparse
+import random
+
+
+parser = argparse.ArgumentParser(description="Prepare mediaspeech")
+parser.add_argument(
+    "--data_path", type=str, help="Path to the directory containing all files"
+)
+parser.add_argument("--train_dir", type=str, help="Path to the train data")
+parser.add_argument("--dev_dir", type=str, help="Path to the dev data")
+parser.add_argument("--test_dir", type=str, help="Path to the test data")
+parser.add_argument("--validated_dir", type=str, help="Path to the validated data")
+parser.add_argument("--dev_ratio", type=float, default=0.136, help="Ratio of dev set")
+parser.add_argument("--test_ratio", type=float, default=0.136, help="Ratio of test set")
+parser.add_argument("--validated_ratio", type=float, default=0.136, help="Ratio of validated set")
+args = parser.parse_args()
+
+file_names = [name[:-5] for name in os.listdir(args.data_path) if name.endswith('.flac')]
+samp_cnt = len(file_names)
+dev_samp_cnt = int(samp_cnt * args.dev_ratio)
+test_samp_cnt = int(samp_cnt * args.test_ratio)
+validated_samp_cnt = int(samp_cnt * args.validated_ratio)
+train_samp_cnt = samp_cnt - dev_samp_cnt - test_samp_cnt - validated_samp_cnt
+
+print ('samp_cnt, dev_samp_cnt, test_samp_cnt, validated_samp_cnt, train_samp_cnt: ', samp_cnt, dev_samp_cnt, test_samp_cnt, validated_samp_cnt, train_samp_cnt)
+print ('file_names: ', file_names[:5])
+
+random.seed(2022)
+random.shuffle(file_names)
+# train_file_names = file_names[:train_samp_cnt]
+# dev_file_names = file_names[train_samp_cnt: train_samp_cnt + dev_samp_cnt]
+# test_file_names = file_names[train_samp_cnt + dev_samp_cnt:]
+
+train_samples = []
+dev_samples = []
+test_samples = []
+validated_samples = []
+
+for file_idx in range(samp_cnt):
+    file_name = file_names[file_idx]
+    text_file_path = os.path.join(args.data_path, file_name + '.txt')
+    aud_file_path = os.path.join(args.data_path, file_name + '.flac')
+
+    with open(text_file_path, 'r') as f:
+        text = f.readlines()[0]
+
+    processed_sample = {
+        'user_id': 0,
+        'text': text,
+        'id': file_idx,
+        'abs_path': 'ffmpeg -i %s -f wav -ar 16000 -ab 16 -ac 1 - |' % os.path.abspath(aud_file_path)
+    }
+    if file_idx < train_samp_cnt:
+        train_samples.append(processed_sample)
+    elif file_idx < train_samp_cnt + dev_samp_cnt:
+        dev_samples.append(processed_sample)
+    elif file_idx < train_samp_cnt + dev_samp_cnt + validated_samp_cnt:
+        validated_samples.append(processed_sample)
+    else:
+        test_samples.append(processed_sample)
+
+for setname in ["train", "dev", "test", "validated"]:
+    if setname == "train":
+        sample_list = train_samples
+        dest_dir = args.train_dir
+    elif setname == "dev":
+        sample_list = dev_samples
+        dest_dir = args.dev_dir
+    elif setname == "test":
+        sample_list = test_samples
+        dest_dir = args.test_dir
+    elif setname == "validated":
+        sample_list = validated_samples
+        dest_dir = args.validated_dir
+    else:
+        raise RuntimeError
+
+    with open(os.path.join(dest_dir, "text"), "w") as text_f, \
+        open(os.path.join(dest_dir, "wav.scp"), "w") as wav_scp_f, \
+        open(os.path.join(dest_dir, "utt2spk"), "w") as utt2spk_f, \
+        open(os.path.join(dest_dir, "utt2gender"), "w") as utt2gndr_f:
+        for sample in sample_list:
+            text_f.write(f"{int(sample['id']):08d} {sample['text']}\n")
+            wav_scp_f.write(f"{int(sample['id']):08d} {sample['abs_path']}\n")
+            utt2spk_f.write(f"{int(sample['id']):08d} {int(sample['id']):08d}\n")
+            utt2gndr_f.write(f"{int(sample['id']):08d}" + " f\n")
+
diff --git a/egs2/mediaspeech/asr1/local/download_and_untar.sh b/egs2/mediaspeech/asr1/local/download_and_untar.sh
new file mode 120000
index 00000000000..794196074a4
--- /dev/null
+++ b/egs2/mediaspeech/asr1/local/download_and_untar.sh
@@ -0,0 +1 @@
+../../../../egs/commonvoice/asr1/local/download_and_untar.sh
\ No newline at end of file
diff --git a/egs2/mediaspeech/asr1/local/path.sh b/egs2/mediaspeech/asr1/local/path.sh
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/egs2/mediaspeech/asr1/path.sh b/egs2/mediaspeech/asr1/path.sh
new file mode 120000
index 00000000000..c9ac0a75bc6
--- /dev/null
+++ b/egs2/mediaspeech/asr1/path.sh
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/path.sh
\ No newline at end of file
diff --git a/egs2/mediaspeech/asr1/pyscripts b/egs2/mediaspeech/asr1/pyscripts
new file mode 120000
index 00000000000..ac68ad75b60
--- /dev/null
+++ b/egs2/mediaspeech/asr1/pyscripts
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/pyscripts
\ No newline at end of file
diff --git a/egs2/mediaspeech/asr1/run.sh b/egs2/mediaspeech/asr1/run.sh
new file mode 100755
index 00000000000..c75a3d23a0b
--- /dev/null
+++ b/egs2/mediaspeech/asr1/run.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+lang=cy # en de fr cy tt kab ca zh-TW it fa eu es ru tr nl eo zh-CN rw pt zh-HK cs pl uk
+
+train_set=train_"$(echo "${lang}" | tr - _)"
+train_dev=dev_"$(echo "${lang}" | tr - _)"
+test_set="${train_dev} test_$(echo ${lang} | tr - _)"
+
+asr_config=conf/tuning/train_asr_conformer5.yaml
+lm_config=conf/train_lm.yaml
+inference_config=conf/decode_asr.yaml
+
+if [[ "zh" == *"${lang}"* ]]; then
+  nbpe=2500
+elif [[ "fr" == *"${lang}"* ]]; then
+  nbpe=350
+elif [[ "es" == *"${lang}"* ]]; then
+  nbpe=235
+else
+  nbpe=150
+fi
+
+./asr.sh \
+    --ngpu 4 \
+    --lang "${lang}" \
+    --local_data_opts "--lang ${lang}" \
+    --use_lm true \
+    --lm_config "${lm_config}" \
+    --token_type bpe \
+    --nbpe $nbpe \
+    --feats_type raw \
+    --speed_perturb_factors "0.9 1.0 1.1" \
+    --asr_config "${asr_config}" \
+    --inference_config "${inference_config}" \
+    --train_set "${train_set}" \
+    --valid_set "${train_dev}" \
+    --test_sets "${test_set}" \
+    --bpe_train_text "data/${train_set}/text" \
+    --lm_train_text "data/${train_set}/text" "$@"
+
diff --git a/egs2/mediaspeech/asr1/scripts b/egs2/mediaspeech/asr1/scripts
new file mode 120000
index 00000000000..b25829705dc
--- /dev/null
+++ b/egs2/mediaspeech/asr1/scripts
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/scripts
\ No newline at end of file
diff --git a/egs2/mediaspeech/asr1/steps b/egs2/mediaspeech/asr1/steps
new file mode 120000
index 00000000000..91f2d234e20
--- /dev/null
+++ b/egs2/mediaspeech/asr1/steps
@@ -0,0 +1 @@
+../../../tools/kaldi/egs/wsj/s5/steps
\ No newline at end of file
diff --git a/egs2/mediaspeech/asr1/utils b/egs2/mediaspeech/asr1/utils
new file mode 120000
index 00000000000..f49247da827
--- /dev/null
+++ b/egs2/mediaspeech/asr1/utils
@@ -0,0 +1 @@
+../../../tools/kaldi/egs/wsj/s5/utils
\ No newline at end of file

From 51bba493adde76ad56e13fa71cf7aaf6e1a991a5 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-22-95.ec2.internal>
Date: Tue, 22 Mar 2022 14:52:39 +0000
Subject: [PATCH 02/56] fix black syntax error and upload hubert result

---
 egs2/TEMPLATE/asr1/db.sh                 |  1 +
 egs2/mediaspeech/asr1/README.md          | 24 +++++++++++++
 egs2/mediaspeech/asr1/local/data_prep.py | 44 +++++++++++++++---------
 3 files changed, 53 insertions(+), 16 deletions(-)

diff --git a/egs2/TEMPLATE/asr1/db.sh b/egs2/TEMPLATE/asr1/db.sh
index 48f2d29a29f..641982d7e2d 100755
--- a/egs2/TEMPLATE/asr1/db.sh
+++ b/egs2/TEMPLATE/asr1/db.sh
@@ -205,6 +205,7 @@ if [[ "$(hostname -d)" == clsp.jhu.edu ]]; then
     FSC=
     SNIPS= # smart-light-en-closed-field data path
     SLURP=
+    MEDIASPEECH=downloads
     MINI_LIBRISPEECH=downloads
     LIBRITTS=
     LJSPEECH=downloads
diff --git a/egs2/mediaspeech/asr1/README.md b/egs2/mediaspeech/asr1/README.md
index e69de29bb2d..53b99998a2d 100644
--- a/egs2/mediaspeech/asr1/README.md
+++ b/egs2/mediaspeech/asr1/README.md
@@ -0,0 +1,24 @@
+<!-- Generated by scripts/utils/show_asr_result.sh -->
+# RESULTS
+## Environments
+- date: `Tue Mar 22 13:50:31 UTC 2022`
+- python version: `3.7.11 (default, Jul 27 2021, 14:32:16)  [GCC 7.5.0]`
+- espnet version: `espnet 0.10.7a1`
+- pytorch version: `pytorch 1.10.1`
+- Git hash: `1991a25855821b8b61d775681aa0cdfd6161bbc8`
+  - Commit date: `Mon Mar 21 22:19:19 2022 +0800`
+
+## asr_train_asr_hubert_raw_as_bpe150_sp
+### WER
+
+|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err|
+|---|---|---|---|---|---|---|---|---|
+|inference_asr_model_valid.acc.ave/dev_as|249|10072|49.7|41.2|9.1|7.0|57.2|100.0|
+|inference_asr_model_valid.acc.ave/test_as|249|9920|51.1|40.1|8.9|6.5|55.4|100.0|
+
+### CER
+
+|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err|
+|---|---|---|---|---|---|---|---|---|
+|inference_asr_model_valid.acc.ave/dev_as|249|58679|80.9|8.0|11.1|7.2|26.3|100.0|
+|inference_asr_model_valid.acc.ave/test_as|249|58694|82.1|7.2|10.8|7.1|25.0|100.0|
diff --git a/egs2/mediaspeech/asr1/local/data_prep.py b/egs2/mediaspeech/asr1/local/data_prep.py
index 59aacfd172e..42162c53da0 100755
--- a/egs2/mediaspeech/asr1/local/data_prep.py
+++ b/egs2/mediaspeech/asr1/local/data_prep.py
@@ -17,18 +17,29 @@
 parser.add_argument("--validated_dir", type=str, help="Path to the validated data")
 parser.add_argument("--dev_ratio", type=float, default=0.136, help="Ratio of dev set")
 parser.add_argument("--test_ratio", type=float, default=0.136, help="Ratio of test set")
-parser.add_argument("--validated_ratio", type=float, default=0.136, help="Ratio of validated set")
+parser.add_argument(
+    "--validated_ratio", type=float, default=0.136, help="Ratio of validated set"
+)
 args = parser.parse_args()
 
-file_names = [name[:-5] for name in os.listdir(args.data_path) if name.endswith('.flac')]
+file_names = [
+    name[:-5] for name in os.listdir(args.data_path) if name.endswith(".flac")
+]
 samp_cnt = len(file_names)
 dev_samp_cnt = int(samp_cnt * args.dev_ratio)
 test_samp_cnt = int(samp_cnt * args.test_ratio)
 validated_samp_cnt = int(samp_cnt * args.validated_ratio)
 train_samp_cnt = samp_cnt - dev_samp_cnt - test_samp_cnt - validated_samp_cnt
 
-print ('samp_cnt, dev_samp_cnt, test_samp_cnt, validated_samp_cnt, train_samp_cnt: ', samp_cnt, dev_samp_cnt, test_samp_cnt, validated_samp_cnt, train_samp_cnt)
-print ('file_names: ', file_names[:5])
+print(
+    "samp_cnt, dev_samp_cnt, test_samp_cnt, validated_samp_cnt, train_samp_cnt: ",
+    samp_cnt,
+    dev_samp_cnt,
+    test_samp_cnt,
+    validated_samp_cnt,
+    train_samp_cnt,
+)
+print("file_names: ", file_names[:5])
 
 random.seed(2022)
 random.shuffle(file_names)
@@ -43,17 +54,18 @@
 
 for file_idx in range(samp_cnt):
     file_name = file_names[file_idx]
-    text_file_path = os.path.join(args.data_path, file_name + '.txt')
-    aud_file_path = os.path.join(args.data_path, file_name + '.flac')
+    text_file_path = os.path.join(args.data_path, file_name + ".txt")
+    aud_file_path = os.path.join(args.data_path, file_name + ".flac")
 
-    with open(text_file_path, 'r') as f:
+    with open(text_file_path, "r") as f:
         text = f.readlines()[0]
 
     processed_sample = {
-        'user_id': 0,
-        'text': text,
-        'id': file_idx,
-        'abs_path': 'ffmpeg -i %s -f wav -ar 16000 -ab 16 -ac 1 - |' % os.path.abspath(aud_file_path)
+        "user_id": 0,
+        "text": text,
+        "id": file_idx,
+        "abs_path": "ffmpeg -i %s -f wav -ar 16000 -ab 16 -ac 1 - |"
+        % os.path.abspath(aud_file_path),
     }
     if file_idx < train_samp_cnt:
         train_samples.append(processed_sample)
@@ -80,13 +92,13 @@
     else:
         raise RuntimeError
 
-    with open(os.path.join(dest_dir, "text"), "w") as text_f, \
-        open(os.path.join(dest_dir, "wav.scp"), "w") as wav_scp_f, \
-        open(os.path.join(dest_dir, "utt2spk"), "w") as utt2spk_f, \
-        open(os.path.join(dest_dir, "utt2gender"), "w") as utt2gndr_f:
+    with open(os.path.join(dest_dir, "text"), "w") as text_f, open(
+        os.path.join(dest_dir, "wav.scp"), "w"
+    ) as wav_scp_f, open(os.path.join(dest_dir, "utt2spk"), "w") as utt2spk_f, open(
+        os.path.join(dest_dir, "utt2gender"), "w"
+    ) as utt2gndr_f:
         for sample in sample_list:
             text_f.write(f"{int(sample['id']):08d} {sample['text']}\n")
             wav_scp_f.write(f"{int(sample['id']):08d} {sample['abs_path']}\n")
             utt2spk_f.write(f"{int(sample['id']):08d} {int(sample['id']):08d}\n")
             utt2gndr_f.write(f"{int(sample['id']):08d}" + " f\n")
-

From 09484d9364ea4ae3f31340b309add6801e6250e2 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-22-95.ec2.internal>
Date: Tue, 22 Mar 2022 15:43:05 +0000
Subject: [PATCH 03/56] fix shellcheck failure

---
 egs2/mediaspeech/asr1/local/data.sh |  4 ----
 egs2/mediaspeech/asr1/run.sh        | 14 ++------------
 2 files changed, 2 insertions(+), 16 deletions(-)

diff --git a/egs2/mediaspeech/asr1/local/data.sh b/egs2/mediaspeech/asr1/local/data.sh
index d5d2aa6d9c9..927c229deb1 100755
--- a/egs2/mediaspeech/asr1/local/data.sh
+++ b/egs2/mediaspeech/asr1/local/data.sh
@@ -36,10 +36,6 @@ set -e
 set -u
 set -o pipefail
 
-train_set=train_"$(echo "${lang}" | tr - _)"
-train_dev=dev_"$(echo "${lang}" | tr - _)"
-test_set=test_"$(echo "${lang}" | tr - _)"
-
 log "data preparation started"
 
 if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then 
diff --git a/egs2/mediaspeech/asr1/run.sh b/egs2/mediaspeech/asr1/run.sh
index c75a3d23a0b..32bb2f49494 100755
--- a/egs2/mediaspeech/asr1/run.sh
+++ b/egs2/mediaspeech/asr1/run.sh
@@ -5,26 +5,16 @@ set -e
 set -u
 set -o pipefail
 
-lang=cy # en de fr cy tt kab ca zh-TW it fa eu es ru tr nl eo zh-CN rw pt zh-HK cs pl uk
+lang=fr
 
 train_set=train_"$(echo "${lang}" | tr - _)"
 train_dev=dev_"$(echo "${lang}" | tr - _)"
 test_set="${train_dev} test_$(echo ${lang} | tr - _)"
 
-asr_config=conf/tuning/train_asr_conformer5.yaml
+asr_config=conf/tuning/train_asr_hubert.yaml
 lm_config=conf/train_lm.yaml
 inference_config=conf/decode_asr.yaml
 
-if [[ "zh" == *"${lang}"* ]]; then
-  nbpe=2500
-elif [[ "fr" == *"${lang}"* ]]; then
-  nbpe=350
-elif [[ "es" == *"${lang}"* ]]; then
-  nbpe=235
-else
-  nbpe=150
-fi
-
 ./asr.sh \
     --ngpu 4 \
     --lang "${lang}" \

From fd0a479fc9be61dc38514561366901608fdebea6 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-22-95.ec2.internal>
Date: Tue, 22 Mar 2022 17:28:05 +0000
Subject: [PATCH 04/56] fix syntax

---
 egs2/mediaspeech/asr1/local/data.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/egs2/mediaspeech/asr1/local/data.sh b/egs2/mediaspeech/asr1/local/data.sh
index 927c229deb1..c4517a8eb47 100755
--- a/egs2/mediaspeech/asr1/local/data.sh
+++ b/egs2/mediaspeech/asr1/local/data.sh
@@ -11,7 +11,6 @@
 stage=0       # start from 0 if you need to start from data preparation
 stop_stage=100
 SECONDS=0
-lang=en # en de fr cy tt kab ca zh-TW it fa eu es ru tr nl eo zh-CN rw pt zh-HK cs pl uk 
 
  . utils/parse_options.sh || exit 1;
 

From cf0bbb10e3a6046f6598988fab62862794126e92 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-22-95.ec2.internal>
Date: Tue, 22 Mar 2022 17:47:28 +0000
Subject: [PATCH 05/56] add TER score

---
 egs2/mediaspeech/asr1/README.md |  7 +++++++
 egs2/mediaspeech/asr1/run.sh    | 12 +++++++++++-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/egs2/mediaspeech/asr1/README.md b/egs2/mediaspeech/asr1/README.md
index 53b99998a2d..80db2d55014 100644
--- a/egs2/mediaspeech/asr1/README.md
+++ b/egs2/mediaspeech/asr1/README.md
@@ -22,3 +22,10 @@
 |---|---|---|---|---|---|---|---|---|
 |inference_asr_model_valid.acc.ave/dev_as|249|58679|80.9|8.0|11.1|7.2|26.3|100.0|
 |inference_asr_model_valid.acc.ave/test_as|249|58694|82.1|7.2|10.8|7.1|25.0|100.0|
+
+### TER
+
+|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err|
+|---|---|---|---|---|---|---|---|---|
+|inference_asr_model_valid.acc.ave/dev_as|249|30837|69.5|19.0|11.5|6.3|36.8|100.0|
+|inference_asr_model_valid.acc.ave/test_as|249|30942|70.7|17.9|11.4|6.0|35.3|100.0|
diff --git a/egs2/mediaspeech/asr1/run.sh b/egs2/mediaspeech/asr1/run.sh
index 32bb2f49494..66cbd0da2ae 100755
--- a/egs2/mediaspeech/asr1/run.sh
+++ b/egs2/mediaspeech/asr1/run.sh
@@ -11,10 +11,20 @@ train_set=train_"$(echo "${lang}" | tr - _)"
 train_dev=dev_"$(echo "${lang}" | tr - _)"
 test_set="${train_dev} test_$(echo ${lang} | tr - _)"
 
-asr_config=conf/tuning/train_asr_hubert.yaml
+asr_config=conf/tuning/train_asr_conformer5.yaml
 lm_config=conf/train_lm.yaml
 inference_config=conf/decode_asr.yaml
 
+if [[ "zh" == *"${lang}"* ]]; then
+  nbpe=2500
+elif [[ "fr" == *"${lang}"* ]]; then
+  nbpe=350
+elif [[ "es" == *"${lang}"* ]]; then
+  nbpe=235
+else
+  nbpe=150
+fi
+
 ./asr.sh \
     --ngpu 4 \
     --lang "${lang}" \

From 7e548c1e8bc194c61b0ce26dd6128340a61c7a97 Mon Sep 17 00:00:00 2001
From: Ashley Wu <wangchew@andrew.cmu.edu>
Date: Wed, 23 Mar 2022 09:46:23 -0400
Subject: [PATCH 06/56] add hf model link abd remove unused code

---
 egs2/README.md                      |  2 +-
 egs2/mediaspeech/asr1/README.md     |  3 +++
 egs2/mediaspeech/asr1/local/data.sh | 14 --------------
 3 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/egs2/README.md b/egs2/README.md
index 5aae03496fb..2e0d55d1ff8 100755
--- a/egs2/README.md
+++ b/egs2/README.md
@@ -54,7 +54,7 @@ See: https://espnet.github.io/espnet/espnet2_tutorial.html#recipes-using-espnet2
 | ljspeech                | The LJ Speech Dataset                                                                   | TTS                     | ENG                  | https://keithito.com/LJ-Speech-Dataset/                                                                      |              |
 | lrs3                    | The Oxford-BBC Lip Reading Sentences 3 (LRS3) Dataset                                   | ASR                     | ENG                  | https://www.robots.ox.ac.uk/~vgg/data/lip_reading/lrs3.html                                                  |              |
 | lrs2                    | The Oxford-BBC Lip Reading Sentences 2 (LRS2) Dataset                                   | Lipreading/ASR          | ENG                  | https://www.robots.ox.ac.uk/~vgg/data/lip_reading/lrs2.html                                                  |              |
-| mediaspeech             | MediaSpeech: Multilanguage ASR Benchmark and Dataset                                    | ASR                     | FR                   | https://www.openslr.org/108/                                                                           |              |
+| mediaspeech             | MediaSpeech: Multilanguage ASR Benchmark and Dataset                                    | ASR                     | FRA                  | https://www.openslr.org/108/                                                                           |              |
 | mini_an4                | Mini version of CMU AN4 database for the integration test                               | ASR/TTS/SE              | ENG                  | http://www.speech.cs.cmu.edu/databases/an4/                                                                  |              |
 | mini_librispeech        | Mini version of Librispeech corpus                                                      | DIAR                    | ENG                  | https://openslr.org/31/                                                                                      |              |
 | mls                     | MLS (A large multilingual corpus derived from LibriVox audiobooks)                      | ASR                     | 8 languages          | http://www.openslr.org/94/                                                                                   |              |
diff --git a/egs2/mediaspeech/asr1/README.md b/egs2/mediaspeech/asr1/README.md
index 80db2d55014..f75dda826d3 100644
--- a/egs2/mediaspeech/asr1/README.md
+++ b/egs2/mediaspeech/asr1/README.md
@@ -1,3 +1,6 @@
+# Hugging Face
+Model is available in Hugging Face: https://huggingface.co/espnet/mediaspeech-fr-hubert
+
 <!-- Generated by scripts/utils/show_asr_result.sh -->
 # RESULTS
 ## Environments
diff --git a/egs2/mediaspeech/asr1/local/data.sh b/egs2/mediaspeech/asr1/local/data.sh
index c4517a8eb47..9a6d54d950e 100755
--- a/egs2/mediaspeech/asr1/local/data.sh
+++ b/egs2/mediaspeech/asr1/local/data.sh
@@ -1,8 +1,5 @@
 #!/usr/bin/env bash
 
-# Copyright 2020 Johns Hopkins University (Shinji Watanabe)
-#  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)
-
 . ./path.sh || exit 1;
 . ./cmd.sh || exit 1;
 . ./db.sh || exit 1;
@@ -15,7 +12,6 @@ SECONDS=0
  . utils/parse_options.sh || exit 1;
 
 # base url for downloads.
-# Deprecated url:https://voice-prod-bundler-ee1969a6ce8178826482b88e843c335139bd3fb4.s3.amazonaws.com/cv-corpus-3/$lang.tar.gz
 data_url=https://us.openslr.org/resources/108/FR.tgz
 
 log() {
@@ -61,16 +57,6 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
         utils/utt2spk_to_spk2utt.pl data/${x}/utt2spk > data/${x}/spk2utt
         utils/fix_data_dir.sh data/${x}
         utils/validate_data_dir.sh --no-feats data/${x}
-    # for part in "validated" "test" "dev"; do
-    #     # use underscore-separated names in data directories.
-    #     local/data_prep.pl "${COMMONVOICE}/cv-corpus-5.1-2020-06-22/${lang}" ${part} data/"$(echo "${part}_${lang}" | tr - _)"
-
-
-    # remove test&dev data from validated sentences
-    # utils/copy_data_dir.sh data/"$(echo "validated_${lang}" | tr - _)" data/${train_set}
-    # utils/filter_scp.pl --exclude data/${train_dev}/wav.scp data/${train_set}/wav.scp > data/${train_set}/temp_wav.scp
-    # utils/filter_scp.pl --exclude data/${test_set}/wav.scp data/${train_set}/temp_wav.scp > data/${train_set}/wav.scp
-    # utils/fix_data_dir.sh data/${train_set}
     done
 fi
 

From c18e8105701ffc20879a3ef21a37bfb19c8a0dd3 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Sun, 10 Apr 2022 23:32:21 +0800
Subject: [PATCH 07/56] Minor adjustments to current enh functions; Add some
 new losses

---
 espnet2/bin/enh_inference.py                  |  4 +-
 espnet2/bin/enh_scoring.py                    |  6 +-
 espnet2/enh/espnet_model.py                   |  7 +-
 espnet2/enh/loss/criterions/tf_domain.py      | 81 +++++++++++++++++--
 espnet2/enh/loss/criterions/time_domain.py    | 62 ++++++++++++++
 espnet2/tasks/enh.py                          |  6 ++
 .../enh/loss/criterions/test_tf_domain.py     | 41 ++++++++--
 .../enh/loss/criterions/test_time_domain.py   |  6 +-
 8 files changed, 194 insertions(+), 19 deletions(-)

diff --git a/espnet2/bin/enh_inference.py b/espnet2/bin/enh_inference.py
index 84a37b5ff7f..f7c5ad797d2 100755
--- a/espnet2/bin/enh_inference.py
+++ b/espnet2/bin/enh_inference.py
@@ -368,13 +368,15 @@ def inference(
     )
 
     # 4. Start for-loop
+    output_dir = Path(output_dir).expanduser().resolve()
     writers = []
     for i in range(separate_speech.num_spk):
         writers.append(
             SoundScpWriter(f"{output_dir}/wavs/{i + 1}", f"{output_dir}/spk{i + 1}.scp")
         )
 
-    for keys, batch in loader:
+    for i, (keys, batch) in enumerate(loader):
+        logging.info(f"[{i}] Enhancing {keys}")
         assert isinstance(batch, dict), type(batch)
         assert all(isinstance(s, str) for s in keys), keys
         _bs = len(next(iter(batch.values())))
diff --git a/espnet2/bin/enh_scoring.py b/espnet2/bin/enh_scoring.py
index a0c9b3c32f6..a8970b2c011 100755
--- a/espnet2/bin/enh_scoring.py
+++ b/espnet2/bin/enh_scoring.py
@@ -79,13 +79,17 @@ def scoring(
 
             for i in range(num_spk):
                 stoi_score = stoi(ref[i], inf[int(perm[i])], fs_sig=sample_rate)
+                estoi_score = stoi(
+                    ref[i], inf[int(perm[i])], fs_sig=sample_rate, extended=True
+                )
                 si_snr_score = -float(
                     si_snr_loss(
                         torch.from_numpy(ref[i][None, ...]),
                         torch.from_numpy(inf[int(perm[i])][None, ...]),
                     )
                 )
-                writer[f"STOI_spk{i + 1}"][key] = str(stoi_score)
+                writer[f"STOI_spk{i + 1}"][key] = str(stoi_score * 100)  # in percentage
+                writer[f"ESTOI_spk{i + 1}"][key] = str(estoi_score * 100)
                 writer[f"SI_SNR_spk{i + 1}"][key] = str(si_snr_score)
                 writer[f"SDR_spk{i + 1}"][key] = str(sdr[i])
                 writer[f"SAR_spk{i + 1}"][key] = str(sar[i])
diff --git a/espnet2/enh/espnet_model.py b/espnet2/enh/espnet_model.py
index f9824471604..04b74d1cfb3 100644
--- a/espnet2/enh/espnet_model.py
+++ b/espnet2/enh/espnet_model.py
@@ -160,10 +160,15 @@ def forward(
             elif isinstance(criterion, FrequencyDomainLoss):
                 # for the time-frequency domain criterions
                 if criterion.compute_on_mask:
-                    # compute on mask
+                    # compute loss on masks
+                    if noise_ref is not None:
+                        noise_spec = self.encoder(noise_ref, speech_lengths)[0]
+                    else:
+                        noise_spec = None
                     tf_ref = criterion.create_mask_label(
                         feature_mix,
                         [self.encoder(sr, speech_lengths)[0] for sr in speech_ref],
+                        noise_spec=noise_spec,
                     )
                     tf_pre = [
                         others["mask_spk{}".format(spk + 1)]
diff --git a/espnet2/enh/loss/criterions/tf_domain.py b/espnet2/enh/loss/criterions/tf_domain.py
index a4d78bfa2f2..e3ca984a197 100644
--- a/espnet2/enh/loss/criterions/tf_domain.py
+++ b/espnet2/enh/loss/criterions/tf_domain.py
@@ -5,6 +5,7 @@
 
 import torch
 
+from espnet2.enh.layers.complex_utils import complex_norm
 from espnet2.enh.layers.complex_utils import is_complex
 from espnet2.enh.layers.complex_utils import new_complex_like
 from espnet2.enh.loss.criterions.abs_loss import AbsEnhLoss
@@ -15,12 +16,14 @@
 EPS = torch.finfo(torch.get_default_dtype()).eps
 
 
-def _create_mask_label(mix_spec, ref_spec, mask_type="IAM"):
+def _create_mask_label(mix_spec, ref_spec, noise_spec=None, mask_type="IAM"):
     """Create mask label.
 
     Args:
         mix_spec: ComplexTensor(B, T, [C,] F)
         ref_spec: List[ComplexTensor(B, T, [C,] F), ...]
+        noise_spec: ComplexTensor(B, T, [C,] F)
+            only used for IBM and IRM
         mask_type: str
     Returns:
         labels: List[Tensor(B, T, [C,] F), ...] or List[ComplexTensor(B, T, F), ...]
@@ -38,16 +41,23 @@ def _create_mask_label(mix_spec, ref_spec, mask_type="IAM"):
         "CIRM",
     ], f"mask type {mask_type} not supported"
     mask_label = []
-    for r in ref_spec:
+    if ref_spec[0].ndim < mix_spec.ndim:
+        # (B, T, F) -> (B, T, 1, F)
+        ref_spec = [r.unsqueeze(2) for r in ref_spec]
+    for idx, r in enumerate(ref_spec):
         mask = None
         if mask_type == "IBM":
-            flags = [abs(r) >= abs(n) for n in ref_spec]
+            if noise_spec is None:
+                raise ValueError("noise_spec should not be None")
+            flags = [abs(r) >= abs(n) for n in ref_spec + [noise_spec]]
             mask = reduce(lambda x, y: x * y, flags)
             mask = mask.int()
         elif mask_type == "IRM":
-            # TODO(Wangyou): need to fix this,
-            #  as noise referecens are provided separately
-            mask = abs(r) / (sum(([abs(n) for n in ref_spec])) + EPS)
+            if noise_spec is None:
+                raise ValueError("noise_spec should not be None")
+            beta = 0.5
+            res_spec = sum(n for i, n in enumerate(ref_spec) if i != idx) + noise_spec
+            mask = (abs(r).pow(2) / (abs(res_spec).pow(2) + EPS)).pow(beta)
         elif mask_type == "IAM":
             mask = abs(r) / (abs(mix_spec) + EPS)
             mask = mask.clamp(min=0, max=1)
@@ -95,9 +105,12 @@ def compute_on_mask() -> bool:
     def mask_type() -> str:
         pass
 
-    def create_mask_label(self, mix_spec, ref_spec):
+    def create_mask_label(self, mix_spec, ref_spec, noise_spec=None):
         return _create_mask_label(
-            mix_spec=mix_spec, ref_spec=ref_spec, mask_type=self.mask_type
+            mix_spec=mix_spec,
+            ref_spec=ref_spec,
+            noise_spec=noise_spec,
+            mask_type=self.mask_type,
         )
 
 
@@ -198,3 +211,55 @@ def forward(self, ref, inf) -> torch.Tensor:
                 "Invalid input shape: ref={}, inf={}".format(ref.shape, inf.shape)
             )
         return l1loss
+
+
+class FrequencyDomainAbsCoherence(FrequencyDomainLoss):
+    def __init__(self, compute_on_mask=False, mask_type=None):
+        super().__init__()
+        self._compute_on_mask = False
+        self._mask_type = None
+
+    @property
+    def compute_on_mask(self) -> bool:
+        return self._compute_on_mask
+
+    @property
+    def mask_type(self) -> str:
+        return self._mask_type
+
+    @property
+    def name(self) -> str:
+        return "Coherence_on_Spec"
+
+    def forward(self, ref, inf) -> torch.Tensor:
+        """time-frequency absolute coherence loss.
+
+        Reference:
+            Independent Vector Analysis with Deep Neural Network Source Priors;
+            Li et al 2020; https://arxiv.org/abs/2008.11273
+
+        Args:
+            ref: (Batch, T, F) or (Batch, T, C, F)
+            inf: (Batch, T, F) or (Batch, T, C, F)
+        Returns:
+            loss: (Batch,)
+        """
+        assert ref.shape == inf.shape, (ref.shape, inf.shape)
+
+        if is_complex(ref) and is_complex(inf):
+            # sqrt( E[|inf|^2] * E[|ref|^2] )
+            denom = (
+                complex_norm(ref, dim=1) * complex_norm(inf, dim=1) / ref.size(1) + EPS
+            )
+            coh = (inf * ref.conj()).mean(dim=1).abs() / denom
+            if ref.dim() == 3:
+                coh_loss = 1.0 - coh.mean(dim=1)
+            elif ref.dim() == 4:
+                coh_loss = 1.0 - coh.mean(dim=[1, 2])
+            else:
+                raise ValueError(
+                    "Invalid input shape: ref={}, inf={}".format(ref.shape, inf.shape)
+                )
+        else:
+            raise ValueError("`ref` and `inf` must be complex tensors.")
+        return coh_loss
diff --git a/espnet2/enh/loss/criterions/time_domain.py b/espnet2/enh/loss/criterions/time_domain.py
index 0ad369d3b44..87d75cfc483 100644
--- a/espnet2/enh/loss/criterions/time_domain.py
+++ b/espnet2/enh/loss/criterions/time_domain.py
@@ -120,3 +120,65 @@ def forward(
         pair_wise_si_snr = 10 * torch.log10(pair_wise_si_snr + self.eps)  # [B]
 
         return -1 * pair_wise_si_snr
+
+
+class TimeDomainMSE(TimeDomainLoss):
+    def __init__(self):
+        super().__init__()
+
+    @property
+    def name(self) -> str:
+        return "time_MSE_loss"
+
+    def forward(self, ref, inf) -> torch.Tensor:
+        """time-domain MSE loss.
+
+        Args:
+            ref: (Batch, T) or (Batch, T, C)
+            inf: (Batch, T) or (Batch, T, C)
+        Returns:
+            loss: (Batch,)
+        """
+        assert ref.shape == inf.shape, (ref.shape, inf.shape)
+
+        mseloss = (ref - inf).pow(2)
+        if ref.dim() == 3:
+            mseloss = mseloss.mean(dim=[1, 2])
+        elif ref.dim() == 2:
+            mseloss = mseloss.mean(dim=1)
+        else:
+            raise ValueError(
+                "Invalid input shape: ref={}, inf={}".format(ref.shape, inf.shape)
+            )
+        return mseloss
+
+
+class TimeDomainL1(TimeDomainLoss):
+    def __init__(self):
+        super().__init__()
+
+    @property
+    def name(self) -> str:
+        return "time_L1_loss"
+
+    def forward(self, ref, inf) -> torch.Tensor:
+        """time-domain L1 loss.
+
+        Args:
+            ref: (Batch, T) or (Batch, T, C)
+            inf: (Batch, T) or (Batch, T, C)
+        Returns:
+            loss: (Batch,)
+        """
+        assert ref.shape == inf.shape, (ref.shape, inf.shape)
+
+        l1loss = abs(ref - inf)
+        if ref.dim() == 3:
+            l1loss = l1loss.mean(dim=[1, 2])
+        elif ref.dim() == 2:
+            l1loss = l1loss.mean(dim=1)
+        else:
+            raise ValueError(
+                "Invalid input shape: ref={}, inf={}".format(ref.shape, inf.shape)
+            )
+        return l1loss
diff --git a/espnet2/tasks/enh.py b/espnet2/tasks/enh.py
index 633bcf1114c..4d50ba18136 100644
--- a/espnet2/tasks/enh.py
+++ b/espnet2/tasks/enh.py
@@ -21,11 +21,14 @@
 from espnet2.enh.encoder.stft_encoder import STFTEncoder
 from espnet2.enh.espnet_model import ESPnetEnhancementModel
 from espnet2.enh.loss.criterions.abs_loss import AbsEnhLoss
+from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainAbsCoherence
 from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainL1
 from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainMSE
 from espnet2.enh.loss.criterions.time_domain import CISDRLoss
 from espnet2.enh.loss.criterions.time_domain import SISNRLoss
 from espnet2.enh.loss.criterions.time_domain import SNRLoss
+from espnet2.enh.loss.criterions.time_domain import TimeDomainL1
+from espnet2.enh.loss.criterions.time_domain import TimeDomainMSE
 from espnet2.enh.loss.wrappers.abs_wrapper import AbsLossWrapper
 from espnet2.enh.loss.wrappers.fixed_order import FixedOrderSolver
 from espnet2.enh.loss.wrappers.pit_solver import PITSolver
@@ -97,8 +100,11 @@
         snr=SNRLoss,
         ci_sdr=CISDRLoss,
         si_snr=SISNRLoss,
+        coh=FrequencyDomainAbsCoherence,
         mse=FrequencyDomainMSE,
         l1=FrequencyDomainL1,
+        mse_t=TimeDomainMSE,
+        l1_t=TimeDomainL1,
     ),
     type_check=AbsEnhLoss,
     default=None,
diff --git a/test/espnet2/enh/loss/criterions/test_tf_domain.py b/test/espnet2/enh/loss/criterions/test_tf_domain.py
index 75e13037217..d188b13f7bf 100644
--- a/test/espnet2/enh/loss/criterions/test_tf_domain.py
+++ b/test/espnet2/enh/loss/criterions/test_tf_domain.py
@@ -1,30 +1,57 @@
+from distutils.version import LooseVersion
 import pytest
 import torch
 
 from torch_complex import ComplexTensor
 
+from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainAbsCoherence
 from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainL1
 from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainMSE
 
 
+is_torch_1_9_plus = LooseVersion(torch.__version__) >= LooseVersion("1.9.0")
+
+
 @pytest.mark.parametrize("criterion_class", [FrequencyDomainL1, FrequencyDomainMSE])
 @pytest.mark.parametrize(
     "mask_type", ["IBM", "IRM", "IAM", "PSM", "NPSM", "PSM^2", "CIRM"]
 )
 @pytest.mark.parametrize("compute_on_mask", [True, False])
-def test_tf_domain_criterion_forward(criterion_class, mask_type, compute_on_mask):
+@pytest.mark.parametrize("input_ch", [1, 2])
+def test_tf_domain_criterion_forward(
+    criterion_class, mask_type, compute_on_mask, input_ch
+):
 
     criterion = criterion_class(compute_on_mask=compute_on_mask, mask_type=mask_type)
+    complex_wrapper = torch.complex if is_torch_1_9_plus else ComplexTensor
 
     batch = 2
-    inf = [torch.rand(batch, 10, 200)]
-    ref_spec = [ComplexTensor(torch.rand(batch, 10, 200), torch.rand(batch, 10, 200))]
-    mix_spec = ComplexTensor(torch.rand(batch, 10, 200), torch.rand(batch, 10, 200))
+    shape = (batch, 10, 200) if input_ch == 1 else (batch, 10, input_ch, 200)
+    ref_spec = [complex_wrapper(torch.rand(*shape), torch.rand(*shape))]
+    mix_spec = complex_wrapper(torch.rand(*shape), torch.rand(*shape))
+    noise_spec = complex_wrapper(torch.rand(*shape), torch.rand(*shape))
 
     if compute_on_mask:
-        ref = criterion.create_mask_label(mix_spec, ref_spec)
+        inf = [torch.rand(*shape)]
+        ref = criterion.create_mask_label(mix_spec, ref_spec, noise_spec=noise_spec)
+        loss = criterion(ref[0], inf[0])
     else:
-        ref = [abs(r) for r in ref_spec]
+        inf_spec = [complex_wrapper(torch.rand(*shape), torch.rand(*shape))]
+        loss = criterion(ref_spec[0], inf_spec[0])
+
+    assert loss.shape == (batch,)
+
+
+@pytest.mark.parametrize("input_ch", [1, 2])
+def test_tf_coh_criterion_forward(input_ch):
+
+    criterion = FrequencyDomainAbsCoherence()
+    complex_wrapper = torch.complex if is_torch_1_9_plus else ComplexTensor
+
+    batch = 2
+    shape = (batch, 10, 200) if input_ch == 1 else (batch, 10, input_ch, 200)
+    inf_spec = complex_wrapper(torch.rand(*shape), torch.rand(*shape))
+    ref_spec = complex_wrapper(torch.rand(*shape), torch.rand(*shape))
 
-    loss = criterion(ref[0], inf[0])
+    loss = criterion(ref_spec, inf_spec)
     assert loss.shape == (batch,)
diff --git a/test/espnet2/enh/loss/criterions/test_time_domain.py b/test/espnet2/enh/loss/criterions/test_time_domain.py
index 250aa604cd0..53f50c88cf6 100644
--- a/test/espnet2/enh/loss/criterions/test_time_domain.py
+++ b/test/espnet2/enh/loss/criterions/test_time_domain.py
@@ -4,9 +4,13 @@
 from espnet2.enh.loss.criterions.time_domain import CISDRLoss
 from espnet2.enh.loss.criterions.time_domain import SISNRLoss
 from espnet2.enh.loss.criterions.time_domain import SNRLoss
+from espnet2.enh.loss.criterions.time_domain import TimeDomainL1
+from espnet2.enh.loss.criterions.time_domain import TimeDomainMSE
 
 
-@pytest.mark.parametrize("criterion_class", [CISDRLoss, SISNRLoss, SNRLoss])
+@pytest.mark.parametrize(
+    "criterion_class", [CISDRLoss, SISNRLoss, SNRLoss, TimeDomainL1, TimeDomainMSE]
+)
 def test_tf_domain_criterion_forward(criterion_class):
 
     criterion = criterion_class()

From 6fc03a2a3fcfef57d6c4a62e5de378efa7f3d336 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Mon, 11 Apr 2022 16:10:22 +0800
Subject: [PATCH 08/56] Fix an issue related to complex_norm argument

---
 espnet2/enh/layers/complex_utils.py           |  9 ++++++---
 test/espnet2/enh/layers/test_complex_utils.py | 10 ++++------
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/espnet2/enh/layers/complex_utils.py b/espnet2/enh/layers/complex_utils.py
index bf4799f58d8..acfbe2f61a8 100644
--- a/espnet2/enh/layers/complex_utils.py
+++ b/espnet2/enh/layers/complex_utils.py
@@ -72,9 +72,12 @@ def complex_norm(
     if is_torch_complex_tensor(c):
         return torch.norm(c, dim=dim, keepdim=keepdim)
     else:
-        return torch.sqrt(
-            (c.real**2 + c.imag**2).sum(dim=dim, keepdim=keepdim) + EPS
-        )
+        if dim is None:
+            return torch.sqrt((c.real**2 + c.imag**2).sum() + EPS)
+        else:
+            return torch.sqrt(
+                (c.real**2 + c.imag**2).sum(dim=dim, keepdim=keepdim) + EPS
+            )
 
 
 def einsum(equation, *operands):
diff --git a/test/espnet2/enh/layers/test_complex_utils.py b/test/espnet2/enh/layers/test_complex_utils.py
index ca988a7bd2b..e566f3aea76 100644
--- a/test/espnet2/enh/layers/test_complex_utils.py
+++ b/test/espnet2/enh/layers/test_complex_utils.py
@@ -52,18 +52,16 @@ def test_cat(dim):
         assert complex_module.allclose(ret, ret2)
 
 
-@pytest.mark.parametrize("dim", [0, 1, 2])
+@pytest.mark.parametrize("dim", [None, 0, 1, 2])
 @pytest.mark.skipif(not is_torch_1_9_plus, reason="Require torch 1.9.0+")
 def test_complex_norm(dim):
     mat = ComplexTensor(torch.rand(2, 3, 4), torch.rand(2, 3, 4))
     mat_th = torch.complex(mat.real, mat.imag)
     norm = complex_norm(mat, dim=dim, keepdim=True)
     norm_th = complex_norm(mat_th, dim=dim, keepdim=True)
-    assert (
-        torch.allclose(norm, norm_th)
-        and norm.ndim == mat.ndim
-        and mat.numel() == norm.numel() * mat.size(dim)
-    )
+    assert torch.allclose(norm, norm_th)
+    if dim is not None:
+        assert norm.ndim == mat.ndim and mat.numel() == norm.numel() * mat.size(dim)
 
 
 @pytest.mark.parametrize("real_vec", [True, False])

From e47667310d5e0ad41bd770d3c429bf1bf32b0393 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Mon, 11 Apr 2022 20:54:22 +0800
Subject: [PATCH 09/56] Fix an issue related to relative path

---
 egs2/TEMPLATE/enh1/scripts/utils/show_enh_score.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/egs2/TEMPLATE/enh1/scripts/utils/show_enh_score.sh b/egs2/TEMPLATE/enh1/scripts/utils/show_enh_score.sh
index 66fb9bc81c2..c0e968ceeca 100755
--- a/egs2/TEMPLATE/enh1/scripts/utils/show_enh_score.sh
+++ b/egs2/TEMPLATE/enh1/scripts/utils/show_enh_score.sh
@@ -15,7 +15,7 @@ fi
 [ -f ./path.sh ] && . ./path.sh
 set -euo pipefail
 if [ $# -eq 1 ]; then
-    exp=$1
+    exp=$(realpath "$1")
 else
     exp=exp
 fi

From 24fe4135dbe3084213a788ae7de9422fe5fcb2b5 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Mon, 11 Apr 2022 22:16:35 +0800
Subject: [PATCH 10/56] Minor fix

---
 egs2/TEMPLATE/enh1/scripts/utils/show_enh_score.sh | 2 +-
 espnet2/enh/encoder/conv_encoder.py                | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/egs2/TEMPLATE/enh1/scripts/utils/show_enh_score.sh b/egs2/TEMPLATE/enh1/scripts/utils/show_enh_score.sh
index c0e968ceeca..1ad8cf14b2d 100755
--- a/egs2/TEMPLATE/enh1/scripts/utils/show_enh_score.sh
+++ b/egs2/TEMPLATE/enh1/scripts/utils/show_enh_score.sh
@@ -51,7 +51,7 @@ while IFS= read -r expdir; do
         metrics=()
         heading="\n|dataset|"
         sep="|---|"
-        for type in pesq stoi sar sdr sir si_snr; do
+        for type in pesq stoi estoi sar sdr sir si_snr; do
             if ls "${expdir}"/*/scoring/result_${type}.txt &> /dev/null; then
                 metrics+=("$type")
                 heading+="${type^^}|"
diff --git a/espnet2/enh/encoder/conv_encoder.py b/espnet2/enh/encoder/conv_encoder.py
index c46f31323e5..c70ccfc164b 100644
--- a/espnet2/enh/encoder/conv_encoder.py
+++ b/espnet2/enh/encoder/conv_encoder.py
@@ -34,7 +34,7 @@ def forward(self, input: torch.Tensor, ilens: torch.Tensor):
         Returns:
             feature (torch.Tensor): mixed feature after encoder [Batch, flens, channel]
         """
-        assert input.dim() == 2, "Currently only support single channle input"
+        assert input.dim() == 2, "Currently only support single channel input"
 
         input = torch.unsqueeze(input, 1)
 

From 21175179a460777e77860966186c0542ceb37489 Mon Sep 17 00:00:00 2001
From: Siddhant <siddhana@br012.ib.bridges2.psc.edu>
Date: Mon, 11 Apr 2022 10:39:50 -0400
Subject: [PATCH 11/56] SLUE Voxpopuli base recipe

---
 egs2/slue-voxpopuli/asr1/README.md            |   2 +
 egs2/slue-voxpopuli/asr1/asr.sh               |   1 +
 egs2/slue-voxpopuli/asr1/cmd.sh               | 110 +++++++++++++
 egs2/slue-voxpopuli/asr1/conf/decode_asr.yaml |   6 +
 .../asr1/conf/decode_asr_low_ctc.yaml         |   6 +
 .../asr1/conf/decode_asr_no_ctc.yaml          |   6 +
 egs2/slue-voxpopuli/asr1/conf/fbank.conf      |   2 +
 egs2/slue-voxpopuli/asr1/conf/pbs.conf        |  11 ++
 egs2/slue-voxpopuli/asr1/conf/pitch.conf      |   1 +
 egs2/slue-voxpopuli/asr1/conf/queue.conf      |  12 ++
 egs2/slue-voxpopuli/asr1/conf/slurm.conf      |  14 ++
 egs2/slue-voxpopuli/asr1/conf/train_asr.yaml  |   1 +
 ..._asr_conformer_lr2e-3_warmup5k_conv2d.yaml |  78 +++++++++
 ...former_lr2e-3_warmup5k_hubert_conv2d2.yaml |  88 +++++++++++
 ...conformer_lr2e-3_warmup5k_tera_conv2d.yaml |  88 +++++++++++
 ...nformer_lr2e-3_warmup5k_vq_apc_conv2d.yaml |  88 +++++++++++
 ...rmer_lr2e-3_warmup5k_wav2vec2_conv2d2.yaml |  88 +++++++++++
 .../asr1/conf/tuning/train_asr_conformer.yaml |  65 ++++++++
 .../conf/tuning/train_asr_conformer_nlu.yaml  |  77 +++++++++
 .../tuning/train_asr_conformer_s3prl.yaml     |  85 ++++++++++
 .../train_asr_wav2vec2_conformer_small.yaml   |  90 +++++++++++
 ...nformer_lr1e-3_warmup5k_wavlm_conv2d2.yaml |  88 +++++++++++
 ...up5k_wav2vec2_conv2d2_wdecay1e-5_bert.yaml |  99 ++++++++++++
 ...5k_wav2vec2_conv2d2_wdecay1e-5_bert_1.yaml | 118 ++++++++++++++
 ...k_wav2vec2_conv2d2_wdecay1e-5_deberta.yaml |  99 ++++++++++++
 ...wav2vec2_conv2d2_wdecay1e-5_deberta_1.yaml | 118 ++++++++++++++
 ...nformer_lr2e-3_warmup5k_wavlm_conv2d2.yaml |  88 +++++++++++
 ...nformer_lr3e-3_warmup5k_wavlm_conv2d2.yaml |  88 +++++++++++
 ...nformer_lr5e-4_warmup5k_wavlm_conv2d2.yaml |  88 +++++++++++
 egs2/slue-voxpopuli/asr1/db.sh                |   1 +
 egs2/slue-voxpopuli/asr1/local/data.sh        |  56 +++++++
 .../local/data_prep_original_slue_format.py   | 122 ++++++++++++++
 ...ta_prep_original_slue_format_transcript.py | 128 +++++++++++++++
 .../asr1/local/data_prep_slue.py              |  66 ++++++++
 egs2/slue-voxpopuli/asr1/local/eval_utils.py  | 149 ++++++++++++++++++
 egs2/slue-voxpopuli/asr1/local/f1_score.py    |  89 +++++++++++
 .../asr1/local/generate_asr_files.py          | 113 +++++++++++++
 egs2/slue-voxpopuli/asr1/local/path.sh        |   0
 egs2/slue-voxpopuli/asr1/local/run_spm.sh     |  38 +++++
 egs2/slue-voxpopuli/asr1/local/score.py       |   1 +
 egs2/slue-voxpopuli/asr1/local/score.sh       |  50 ++++++
 egs2/slue-voxpopuli/asr1/path.sh              |   1 +
 egs2/slue-voxpopuli/asr1/pyscripts            |   1 +
 egs2/slue-voxpopuli/asr1/run.sh               |  33 ++++
 egs2/slue-voxpopuli/asr1/scripts              |   1 +
 egs2/slue-voxpopuli/asr1/steps                |   1 +
 egs2/slue-voxpopuli/asr1/utils                |   1 +
 47 files changed, 2556 insertions(+)
 create mode 100644 egs2/slue-voxpopuli/asr1/README.md
 create mode 120000 egs2/slue-voxpopuli/asr1/asr.sh
 create mode 100644 egs2/slue-voxpopuli/asr1/cmd.sh
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/decode_asr.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/decode_asr_low_ctc.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/decode_asr_no_ctc.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/fbank.conf
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/pbs.conf
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/pitch.conf
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/queue.conf
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/slurm.conf
 create mode 120000 egs2/slue-voxpopuli/asr1/conf/train_asr.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_conv2d.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_hubert_conv2d2.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_tera_conv2d.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_vq_apc_conv2d.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer_nlu.yaml
 create mode 100755 egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer_s3prl.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_wav2vec2_conformer_small.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr1e-3_warmup5k_wavlm_conv2d2.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_bert.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_bert_1.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_deberta.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_deberta_1.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wavlm_conv2d2.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr3e-3_warmup5k_wavlm_conv2d2.yaml
 create mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr5e-4_warmup5k_wavlm_conv2d2.yaml
 create mode 120000 egs2/slue-voxpopuli/asr1/db.sh
 create mode 100755 egs2/slue-voxpopuli/asr1/local/data.sh
 create mode 100644 egs2/slue-voxpopuli/asr1/local/data_prep_original_slue_format.py
 create mode 100644 egs2/slue-voxpopuli/asr1/local/data_prep_original_slue_format_transcript.py
 create mode 100644 egs2/slue-voxpopuli/asr1/local/data_prep_slue.py
 create mode 100644 egs2/slue-voxpopuli/asr1/local/eval_utils.py
 create mode 100755 egs2/slue-voxpopuli/asr1/local/f1_score.py
 create mode 100644 egs2/slue-voxpopuli/asr1/local/generate_asr_files.py
 create mode 100644 egs2/slue-voxpopuli/asr1/local/path.sh
 create mode 100755 egs2/slue-voxpopuli/asr1/local/run_spm.sh
 create mode 120000 egs2/slue-voxpopuli/asr1/local/score.py
 create mode 100755 egs2/slue-voxpopuli/asr1/local/score.sh
 create mode 120000 egs2/slue-voxpopuli/asr1/path.sh
 create mode 120000 egs2/slue-voxpopuli/asr1/pyscripts
 create mode 100755 egs2/slue-voxpopuli/asr1/run.sh
 create mode 120000 egs2/slue-voxpopuli/asr1/scripts
 create mode 120000 egs2/slue-voxpopuli/asr1/steps
 create mode 120000 egs2/slue-voxpopuli/asr1/utils

diff --git a/egs2/slue-voxpopuli/asr1/README.md b/egs2/slue-voxpopuli/asr1/README.md
new file mode 100644
index 00000000000..139597f9cb0
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/README.md
@@ -0,0 +1,2 @@
+
+
diff --git a/egs2/slue-voxpopuli/asr1/asr.sh b/egs2/slue-voxpopuli/asr1/asr.sh
new file mode 120000
index 00000000000..60b05122cfd
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/asr.sh
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/asr.sh
\ No newline at end of file
diff --git a/egs2/slue-voxpopuli/asr1/cmd.sh b/egs2/slue-voxpopuli/asr1/cmd.sh
new file mode 100644
index 00000000000..2aae6919fef
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/cmd.sh
@@ -0,0 +1,110 @@
+# ====== About run.pl, queue.pl, slurm.pl, and ssh.pl ======
+# Usage: <cmd>.pl [options] JOB=1:<nj> <log> <command...>
+# e.g.
+#   run.pl --mem 4G JOB=1:10 echo.JOB.log echo JOB
+#
+# Options:
+#   --time <time>: Limit the maximum time to execute.
+#   --mem <mem>: Limit the maximum memory usage.
+#   -–max-jobs-run <njob>: Limit the number parallel jobs. This is ignored for non-array jobs.
+#   --num-threads <ngpu>: Specify the number of CPU core.
+#   --gpu <ngpu>: Specify the number of GPU devices.
+#   --config: Change the configuration file from default.
+#
+# "JOB=1:10" is used for "array jobs" and it can control the number of parallel jobs.
+# The left string of "=", i.e. "JOB", is replaced by <N>(Nth job) in the command and the log file name,
+# e.g. "echo JOB" is changed to "echo 3" for the 3rd job and "echo 8" for 8th job respectively.
+# Note that the number must start with a positive number, so you can't use "JOB=0:10" for example.
+#
+# run.pl, queue.pl, slurm.pl, and ssh.pl have unified interface, not depending on its backend.
+# These options are mapping to specific options for each backend and
+# it is configured by "conf/queue.conf" and "conf/slurm.conf" by default.
+# If jobs failed, your configuration might be wrong for your environment.
+#
+#
+# The official documentation for run.pl, queue.pl, slurm.pl, and ssh.pl:
+#   "Parallelization in Kaldi": http://kaldi-asr.org/doc/queue.html
+# =========================================================~
+
+
+# Select the backend used by run.sh from "local", "stdout", "sge", "slurm", or "ssh"
+cmd_backend='local'
+
+# Local machine, without any Job scheduling system
+if [ "${cmd_backend}" = local ]; then
+
+    # The other usage
+    export train_cmd="run.pl"
+    # Used for "*_train.py": "--gpu" is appended optionally by run.sh
+    export cuda_cmd="run.pl"
+    # Used for "*_recog.py"
+    export decode_cmd="run.pl"
+
+# Local machine logging to stdout and log file, without any Job scheduling system
+elif [ "${cmd_backend}" = stdout ]; then
+
+    # The other usage
+    export train_cmd="stdout.pl"
+    # Used for "*_train.py": "--gpu" is appended optionally by run.sh
+    export cuda_cmd="stdout.pl"
+    # Used for "*_recog.py"
+    export decode_cmd="stdout.pl"
+
+
+# "qsub" (Sun Grid Engine, or derivation of it)
+elif [ "${cmd_backend}" = sge ]; then
+    # The default setting is written in conf/queue.conf.
+    # You must change "-q g.q" for the "queue" for your environment.
+    # To know the "queue" names, type "qhost -q"
+    # Note that to use "--gpu *", you have to setup "complex_value" for the system scheduler.
+
+    export train_cmd="queue.pl"
+    export cuda_cmd="queue.pl"
+    export decode_cmd="queue.pl"
+
+
+# "qsub" (Torque/PBS.)
+elif [ "${cmd_backend}" = pbs ]; then
+    # The default setting is written in conf/pbs.conf.
+
+    export train_cmd="pbs.pl"
+    export cuda_cmd="pbs.pl"
+    export decode_cmd="pbs.pl"
+
+
+# "sbatch" (Slurm)
+elif [ "${cmd_backend}" = slurm ]; then
+    # The default setting is written in conf/slurm.conf.
+    # You must change "-p cpu" and "-p gpu" for the "partition" for your environment.
+    # To know the "partion" names, type "sinfo".
+    # You can use "--gpu * " by default for slurm and it is interpreted as "--gres gpu:*"
+    # The devices are allocated exclusively using "${CUDA_VISIBLE_DEVICES}".
+
+    export train_cmd="slurm.pl"
+    export cuda_cmd="slurm.pl"
+    export decode_cmd="slurm.pl"
+
+elif [ "${cmd_backend}" = ssh ]; then
+    # You have to create ".queue/machines" to specify the host to execute jobs.
+    # e.g. .queue/machines
+    #   host1
+    #   host2
+    #   host3
+    # Assuming you can login them without any password, i.e. You have to set ssh keys.
+
+    export train_cmd="ssh.pl"
+    export cuda_cmd="ssh.pl"
+    export decode_cmd="ssh.pl"
+
+# This is an example of specifying several unique options in the JHU CLSP cluster setup.
+# Users can modify/add their own command options according to their cluster environments.
+elif [ "${cmd_backend}" = jhu ]; then
+
+    export train_cmd="queue.pl --mem 2G"
+    export cuda_cmd="queue-freegpu.pl --mem 2G --gpu 1 --config conf/queue.conf"
+    export decode_cmd="queue.pl --mem 4G"
+
+else
+    echo "$0: Error: Unknown cmd_backend=${cmd_backend}" 1>&2
+    return 1
+fi
diff --git a/egs2/slue-voxpopuli/asr1/conf/decode_asr.yaml b/egs2/slue-voxpopuli/asr1/conf/decode_asr.yaml
new file mode 100644
index 00000000000..4a05b1fae3b
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/decode_asr.yaml
@@ -0,0 +1,6 @@
+beam_size: 10
+ctc_weight: 0.3
+lm_weight: 0.0
+maxlenratio: 0.0
+minlenratio: 0.0
+penalty: 0.0
diff --git a/egs2/slue-voxpopuli/asr1/conf/decode_asr_low_ctc.yaml b/egs2/slue-voxpopuli/asr1/conf/decode_asr_low_ctc.yaml
new file mode 100644
index 00000000000..7aa1597192c
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/decode_asr_low_ctc.yaml
@@ -0,0 +1,6 @@
+beam_size: 10
+ctc_weight: 0.1
+lm_weight: 0.0
+maxlenratio: 0.0
+minlenratio: 0.0
+penalty: 0.0
diff --git a/egs2/slue-voxpopuli/asr1/conf/decode_asr_no_ctc.yaml b/egs2/slue-voxpopuli/asr1/conf/decode_asr_no_ctc.yaml
new file mode 100644
index 00000000000..a86d524e203
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/decode_asr_no_ctc.yaml
@@ -0,0 +1,6 @@
+beam_size: 10
+ctc_weight: 0.0
+lm_weight: 0.0
+maxlenratio: 0.0
+minlenratio: 0.0
+penalty: 0.0
diff --git a/egs2/slue-voxpopuli/asr1/conf/fbank.conf b/egs2/slue-voxpopuli/asr1/conf/fbank.conf
new file mode 100644
index 00000000000..82ac7bd0dbc
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/fbank.conf
@@ -0,0 +1,2 @@
+--sample-frequency=16000 
+--num-mel-bins=80
diff --git a/egs2/slue-voxpopuli/asr1/conf/pbs.conf b/egs2/slue-voxpopuli/asr1/conf/pbs.conf
new file mode 100644
index 00000000000..119509938ce
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/pbs.conf
@@ -0,0 +1,11 @@
+# Default configuration
+command qsub -V -v PATH -S /bin/bash
+option name=* -N $0
+option mem=* -l mem=$0
+option mem=0          # Do not add anything to qsub_opts
+option num_threads=* -l ncpus=$0
+option num_threads=1  # Do not add anything to qsub_opts
+option num_nodes=* -l nodes=$0:ppn=1
+default gpu=0
+option gpu=0
+option gpu=* -l ngpus=$0
diff --git a/egs2/slue-voxpopuli/asr1/conf/pitch.conf b/egs2/slue-voxpopuli/asr1/conf/pitch.conf
new file mode 100644
index 00000000000..e959a19d5b8
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/pitch.conf
@@ -0,0 +1 @@
+--sample-frequency=16000
diff --git a/egs2/slue-voxpopuli/asr1/conf/queue.conf b/egs2/slue-voxpopuli/asr1/conf/queue.conf
new file mode 100644
index 00000000000..500582fab31
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/queue.conf
@@ -0,0 +1,12 @@
+# Default configuration
+command qsub -v PATH -cwd -S /bin/bash -j y -l arch=*64*
+option name=* -N $0
+option mem=* -l mem_free=$0,ram_free=$0
+option mem=0          # Do not add anything to qsub_opts
+option num_threads=* -pe smp $0
+option num_threads=1  # Do not add anything to qsub_opts
+option max_jobs_run=* -tc $0
+option num_nodes=* -pe mpi $0  # You must set this PE as allocation_rule=1
+default gpu=0
+option gpu=0
+option gpu=* -l gpu=$0 -q g.q
diff --git a/egs2/slue-voxpopuli/asr1/conf/slurm.conf b/egs2/slue-voxpopuli/asr1/conf/slurm.conf
new file mode 100644
index 00000000000..3b229673638
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/slurm.conf
@@ -0,0 +1,14 @@
+# Default configuration
+command sbatch --export=PATH
+option name=* --job-name $0
+option time=* --time $0
+option mem=* --mem-per-cpu $0
+option mem=0
+option num_threads=* --cpus-per-task $0
+option num_threads=1 --cpus-per-task 1
+option num_nodes=* --nodes $0
+default gpu=0
+option gpu=0 -p cpu
+option gpu=* -p gpu --gres=gpu:$0 -c $0  # Recommend allocating more CPU than, or equal to the number of GPU
+# note: the --max-jobs-run option is supported as a special case
+# by slurm.pl and you don't have to handle it in the config file.
diff --git a/egs2/slue-voxpopuli/asr1/conf/train_asr.yaml b/egs2/slue-voxpopuli/asr1/conf/train_asr.yaml
new file mode 120000
index 00000000000..b79a2904e79
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/train_asr.yaml
@@ -0,0 +1 @@
+tuning/train_asr_wav2vec2_conformer_small.yaml
\ No newline at end of file
diff --git a/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_conv2d.yaml b/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_conv2d.yaml
new file mode 100644
index 00000000000..4f3a1ff1917
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_conv2d.yaml
@@ -0,0 +1,78 @@
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 27
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_ratio_range:
+    - 0.
+    - 0.05
+    num_time_mask: 2
+
+encoder: conformer
+encoder_conf:
+    output_size: 256
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d
+    normalize_before: true
+    macaron_style: true
+    rel_pos_type: latest
+    pos_enc_layer_type: rel_pos
+    selfattention_layer_type: rel_selfattn
+    activation_type: swish
+    use_cnn_module: true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
+
+frontend_conf:
+    n_fft: 512
+    win_length: 400
+    hop_length: 160
+
+seed: 2022
+log_interval: 100
+num_workers: 2
+batch_type: numel
+batch_bins: 6000000
+accum_grad: 2
+max_epoch: 70
+patience: none
+init: none
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+    lr: 0.002
+    weight_decay: 0.000001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_hubert_conv2d2.yaml b/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_hubert_conv2d2.yaml
new file mode 100644
index 00000000000..0de49db951b
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_hubert_conv2d2.yaml
@@ -0,0 +1,88 @@
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 27
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_ratio_range:
+    - 0.
+    - 0.05
+    num_time_mask: 2
+
+encoder: conformer
+encoder_conf:
+    output_size: 256
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d2
+    normalize_before: true
+    macaron_style: true
+    rel_pos_type: latest
+    pos_enc_layer_type: rel_pos
+    selfattention_layer_type: rel_selfattn
+    activation_type: swish
+    use_cnn_module: true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
+
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream: hubert_large_ll60k  # Note: If the upstream is changed, please change the input_size in the preencoder.
+    download_dir: ./hub
+    multilayer_feature: true
+
+preencoder: linear
+preencoder_conf:
+    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+freeze_param:
+    - "frontend.upstream"
+
+seed: 2022
+log_interval: 100
+num_workers: 2
+batch_type: numel
+batch_bins: 6000000
+accum_grad: 2
+max_epoch: 70
+patience: none
+init: none
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+    lr: 0.002
+    weight_decay: 0.000001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_tera_conv2d.yaml b/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_tera_conv2d.yaml
new file mode 100644
index 00000000000..bde54149c47
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_tera_conv2d.yaml
@@ -0,0 +1,88 @@
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 27
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_ratio_range:
+    - 0.
+    - 0.05
+    num_time_mask: 2
+
+encoder: conformer
+encoder_conf:
+    output_size: 256
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d
+    normalize_before: true
+    macaron_style: true
+    rel_pos_type: latest
+    pos_enc_layer_type: rel_pos
+    selfattention_layer_type: rel_selfattn
+    activation_type: swish
+    use_cnn_module: true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
+
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream: tera_960hr  # Note: If the upstream is changed, please change the input_size in the preencoder.
+    download_dir: ./hub
+    multilayer_feature: true
+
+preencoder: linear
+preencoder_conf:
+    input_size: 768  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+freeze_param:
+    - "frontend.upstream"
+
+seed: 2022
+log_interval: 100
+num_workers: 2
+batch_type: numel
+batch_bins: 6000000
+accum_grad: 2
+max_epoch: 70
+patience: none
+init: none
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+    lr: 0.002
+    weight_decay: 0.000001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_vq_apc_conv2d.yaml b/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_vq_apc_conv2d.yaml
new file mode 100644
index 00000000000..0d17cd8cef0
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_vq_apc_conv2d.yaml
@@ -0,0 +1,88 @@
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 27
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_ratio_range:
+    - 0.
+    - 0.05
+    num_time_mask: 2
+
+encoder: conformer
+encoder_conf:
+    output_size: 256
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d
+    normalize_before: true
+    macaron_style: true
+    rel_pos_type: latest
+    pos_enc_layer_type: rel_pos
+    selfattention_layer_type: rel_selfattn
+    activation_type: swish
+    use_cnn_module: true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
+
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream: vq_apc_960hr  # Note: If the upstream is changed, please change the input_size in the preencoder.
+    download_dir: ./hub
+    multilayer_feature: true
+
+preencoder: linear
+preencoder_conf:
+    input_size: 512  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+freeze_param:
+    - "frontend.upstream"
+
+seed: 2022
+log_interval: 100
+num_workers: 2
+batch_type: numel
+batch_bins: 6000000
+accum_grad: 2
+max_epoch: 70
+patience: none
+init: none
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+    lr: 0.002
+    weight_decay: 0.000001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2.yaml b/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2.yaml
new file mode 100644
index 00000000000..eb25f604f1e
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2.yaml
@@ -0,0 +1,88 @@
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 27
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_ratio_range:
+    - 0.
+    - 0.05
+    num_time_mask: 2
+
+encoder: conformer
+encoder_conf:
+    output_size: 256
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d2
+    normalize_before: true
+    macaron_style: true
+    rel_pos_type: latest
+    pos_enc_layer_type: rel_pos
+    selfattention_layer_type: rel_selfattn
+    activation_type: swish
+    use_cnn_module: true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
+
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream: wav2vec2_large_ll60k  # Note: If the upstream is changed, please change the input_size in the preencoder.
+    download_dir: ./hub
+    multilayer_feature: true
+
+preencoder: linear
+preencoder_conf:
+    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+freeze_param:
+    - "frontend.upstream"
+
+seed: 2022
+log_interval: 100
+num_workers: 2
+batch_type: numel
+batch_bins: 6000000
+accum_grad: 2
+max_epoch: 70
+patience: none
+init: none
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+    lr: 0.002
+    weight_decay: 0.000001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer.yaml
new file mode 100644
index 00000000000..ee983ed0e8c
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer.yaml
@@ -0,0 +1,65 @@
+# network architecture
+# encoder related
+encoder: conformer
+encoder_conf:
+    output_size: 512
+    attention_heads: 8
+    linear_units: 2048
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d
+    normalize_before: true
+    macaron_style: true
+    pos_enc_layer_type: "rel_pos"
+    selfattention_layer_type: "rel_selfattn"
+    activation_type: "swish"
+    use_cnn_module:  true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 8
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+optim: adam
+optim_conf:
+    lr: 0.0002
+scheduler: warmuplr     # pytorch v1.1.0+ required #Tune warmup steps
+scheduler_conf:
+    warmup_steps: 25000
+max_epoch: 50
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: "False" means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend.
+
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 40
+    num_time_mask: 2
+
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
\ No newline at end of file
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer_nlu.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer_nlu.yaml
new file mode 100644
index 00000000000..a10d58445aa
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer_nlu.yaml
@@ -0,0 +1,77 @@
+# network architecture
+# encoder related
+encoder: conformer
+encoder_conf:
+    output_size: 512
+    attention_heads: 8
+    linear_units: 2048
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d
+    normalize_before: true
+    macaron_style: true
+    pos_enc_layer_type: "rel_pos"
+    selfattention_layer_type: "rel_selfattn"
+    activation_type: "swish"
+    use_cnn_module:  true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 8
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+postencoder: hugging_face_transformers
+postencoder_conf:
+    # pick up a model from https://huggingface.co/models?filter=transformers
+    # most of models should work, but maybe some don't
+    # known to work: bert, gpt2, xlnet, roberta, mpnet, t5, bart
+    # xlnet currently works for single gpu only
+    model_name_or_path: "bert-base-uncased"
+
+freeze_param: [
+"postencoder.transformer"
+]
+
+optim: adam
+optim_conf:
+    lr: 0.0002
+scheduler: warmuplr     # pytorch v1.1.0+ required #Tune warmup steps
+scheduler_conf:
+    warmup_steps: 25000
+max_epoch: 50
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: "False" means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend.
+
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 40
+    num_time_mask: 2
+
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer_s3prl.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer_s3prl.yaml
new file mode 100755
index 00000000000..c6acbbb652a
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer_s3prl.yaml
@@ -0,0 +1,85 @@
+# network architecture
+# encoder related
+encoder: conformer
+encoder_conf:
+    output_size: 512
+    attention_heads: 8
+    linear_units: 2048
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d
+    normalize_before: true
+    macaron_style: true
+    pos_enc_layer_type: "rel_pos"
+    selfattention_layer_type: "rel_selfattn"
+    activation_type: "swish"
+    use_cnn_module:  true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 8
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+optim: adam
+optim_conf:
+    lr: 0.0002
+scheduler: warmuplr     # pytorch v1.1.0+ required
+scheduler_conf:
+    warmup_steps: 25000
+max_epoch: 50
+
+freeze_param: [
+"frontend.upstream"
+]
+
+frontend_conf:
+  n_fft: 512
+  hop_length: 256
+
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream: hubert_large_ll60k  # Note: If the upstream is changed, please change the input_size in the preencoder.
+    download_dir: ./hub
+    multilayer_feature: True
+
+preencoder: linear
+preencoder_conf:
+    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: "False" means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend.
+
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 40
+    num_time_mask: 2
+
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
\ No newline at end of file
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_wav2vec2_conformer_small.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_wav2vec2_conformer_small.yaml
new file mode 100644
index 00000000000..fe8f902cb63
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_wav2vec2_conformer_small.yaml
@@ -0,0 +1,90 @@
+# network architecture
+# encoder related
+batch_type: numel
+batch_bins: 6000000
+encoder: conformer
+accum_grad: 2
+
+encoder_conf:
+    output_size: 256
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d2
+    normalize_before: true
+    macaron_style: true
+    pos_enc_layer_type: "rel_pos"
+    selfattention_layer_type: "rel_selfattn"
+    activation_type: "swish"
+    use_cnn_module:  true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+optim: adam
+optim_conf:
+    lr: 0.002
+    weight_decay: 1.0e-06
+scheduler: warmuplr     # pytorch v1.1.0+ required #Tune warmup steps
+scheduler_conf:
+    warmup_steps: 5000
+max_epoch: 100
+
+freeze_param: [
+"frontend.upstream"
+]
+
+frontend_conf:
+  n_fft: 512
+  hop_length: 256
+
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream: wav2vec2_large_ll60k  # Note: If the upstream is changed, please change the input_size in the preencoder.
+    download_dir: ./hub
+    multilayer_feature: True
+
+preencoder: linear
+preencoder_conf:
+    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: "False" means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend.
+
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 40
+    num_time_mask: 2
+
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr1e-3_warmup5k_wavlm_conv2d2.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr1e-3_warmup5k_wavlm_conv2d2.yaml
new file mode 100644
index 00000000000..842684710a2
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr1e-3_warmup5k_wavlm_conv2d2.yaml
@@ -0,0 +1,88 @@
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 27
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_ratio_range:
+    - 0.
+    - 0.05
+    num_time_mask: 2
+
+encoder: conformer
+encoder_conf:
+    output_size: 256
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d2
+    normalize_before: true
+    macaron_style: true
+    rel_pos_type: latest
+    pos_enc_layer_type: rel_pos
+    selfattention_layer_type: rel_selfattn
+    activation_type: swish
+    use_cnn_module: true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
+
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream: wavlm_large  # Note: If the upstream is changed, please change the input_size in the preencoder.
+    download_dir: ./hub
+    multilayer_feature: true
+
+preencoder: linear
+preencoder_conf:
+    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+freeze_param:
+    - "frontend.upstream"
+
+seed: 2022
+log_interval: 100
+num_workers: 2
+batch_type: numel
+batch_bins: 6000000
+accum_grad: 2
+max_epoch: 70
+patience: none
+init: none
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+    lr: 0.001
+    weight_decay: 0.000001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_bert.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_bert.yaml
new file mode 100644
index 00000000000..311b9e8a562
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_bert.yaml
@@ -0,0 +1,99 @@
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 27
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_ratio_range:
+    - 0.
+    - 0.05
+    num_time_mask: 2
+
+encoder: conformer
+encoder_conf:
+    output_size: 256
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d2
+    normalize_before: true
+    macaron_style: true
+    rel_pos_type: latest
+    pos_enc_layer_type: rel_pos
+    selfattention_layer_type: rel_selfattn
+    activation_type: swish
+    use_cnn_module: true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
+
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream:  wavlm_large  # Note: If the upstream is changed, please change the input_size in the preencoder.
+    download_dir: ./hub
+    multilayer_feature: true
+
+preencoder: linear
+preencoder_conf:
+    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+postdecoder: hugging_face_transformers
+postdecoder_conf:
+    # pick up a model from https://huggingface.co/models?filter=transformers
+    # most of models should work, but maybe some don't
+    # known to work: bert, gpt2, xlnet, roberta, mpnet, t5, bart
+    # xlnet currently works for single gpu only
+    model_name_or_path: "bert-base-cased"
+    output_size: 256
+
+freeze_param:
+    - "frontend.upstream"
+    - "encoder"
+    - "postdecoder.model"
+
+seed: 2022
+log_interval: 100
+num_workers: 2
+batch_type: numel
+batch_bins: 6000000
+accum_grad: 2
+max_epoch: 70
+patience: none
+init: none
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+    lr: 0.002
+    weight_decay: 0.000001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_bert_1.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_bert_1.yaml
new file mode 100644
index 00000000000..45f9a96b01f
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_bert_1.yaml
@@ -0,0 +1,118 @@
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 27
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_ratio_range:
+    - 0.
+    - 0.05
+    num_time_mask: 2
+
+encoder: conformer
+encoder_conf:
+    output_size: 256
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d2
+    normalize_before: true
+    macaron_style: true
+    rel_pos_type: latest
+    pos_enc_layer_type: rel_pos
+    selfattention_layer_type: rel_selfattn
+    activation_type: swish
+    use_cnn_module: true
+    cnn_module_kernel: 31
+
+deliberationencoder: conformer
+deliberationencoder_conf:
+    output_size: 256
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: linear
+    normalize_before: true
+    macaron_style: true
+    rel_pos_type: latest
+    pos_enc_layer_type: rel_pos
+    selfattention_layer_type: rel_selfattn
+    activation_type: swish
+    use_cnn_module: true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
+
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream:  wavlm_large  # Note: If the upstream is changed, please change the input_size in the preencoder.
+    download_dir: ./hub
+    multilayer_feature: true
+
+preencoder: linear
+preencoder_conf:
+    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+postdecoder: hugging_face_transformers
+postdecoder_conf:
+    # pick up a model from https://huggingface.co/models?filter=transformers
+    # most of models should work, but maybe some don't
+    # known to work: bert, gpt2, xlnet, roberta, mpnet, t5, bart
+    # xlnet currently works for single gpu only
+    model_name_or_path: "bert-base-cased"
+    output_size: 256
+
+freeze_param:
+    - "frontend.upstream"
+    - "encoder"
+    - "postdecoder.model"
+
+seed: 2022
+log_interval: 100
+num_workers: 2
+batch_type: numel
+batch_bins: 6000000
+accum_grad: 2
+max_epoch: 65
+patience: none
+init: none
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+    lr: 0.002
+    weight_decay: 0.000001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_deberta.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_deberta.yaml
new file mode 100644
index 00000000000..081b08c1ea6
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_deberta.yaml
@@ -0,0 +1,99 @@
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 27
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_ratio_range:
+    - 0.
+    - 0.05
+    num_time_mask: 2
+
+encoder: conformer
+encoder_conf:
+    output_size: 256
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d2
+    normalize_before: true
+    macaron_style: true
+    rel_pos_type: latest
+    pos_enc_layer_type: rel_pos
+    selfattention_layer_type: rel_selfattn
+    activation_type: swish
+    use_cnn_module: true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
+
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream:  wavlm_large  # Note: If the upstream is changed, please change the input_size in the preencoder.
+    download_dir: ./hub
+    multilayer_feature: true
+
+preencoder: linear
+preencoder_conf:
+    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+postdecoder: hugging_face_transformers
+postdecoder_conf:
+    # pick up a model from https://huggingface.co/models?filter=transformers
+    # most of models should work, but maybe some don't
+    # known to work: bert, gpt2, xlnet, roberta, mpnet, t5, bart
+    # xlnet currently works for single gpu only
+    model_name_or_path: "microsoft/deberta-large"
+    output_size: 256
+
+freeze_param:
+    - "frontend.upstream"
+    - "encoder"
+    - "postdecoder.model"
+
+seed: 2022
+log_interval: 100
+num_workers: 2
+batch_type: numel
+batch_bins: 6000000
+accum_grad: 2
+max_epoch: 60
+patience: none
+init: none
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+    lr: 0.002
+    weight_decay: 0.000001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_deberta_1.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_deberta_1.yaml
new file mode 100644
index 00000000000..7bd8c40432b
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_deberta_1.yaml
@@ -0,0 +1,118 @@
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 27
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_ratio_range:
+    - 0.
+    - 0.05
+    num_time_mask: 2
+
+encoder: conformer
+encoder_conf:
+    output_size: 256
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d2
+    normalize_before: true
+    macaron_style: true
+    rel_pos_type: latest
+    pos_enc_layer_type: rel_pos
+    selfattention_layer_type: rel_selfattn
+    activation_type: swish
+    use_cnn_module: true
+    cnn_module_kernel: 31
+
+deliberationencoder: conformer
+deliberationencoder_conf:
+    output_size: 256
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: linear
+    normalize_before: true
+    macaron_style: true
+    rel_pos_type: latest
+    pos_enc_layer_type: rel_pos
+    selfattention_layer_type: rel_selfattn
+    activation_type: swish
+    use_cnn_module: true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
+
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream:  wavlm_large  # Note: If the upstream is changed, please change the input_size in the preencoder.
+    download_dir: ./hub
+    multilayer_feature: true
+
+preencoder: linear
+preencoder_conf:
+    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+postdecoder: hugging_face_transformers
+postdecoder_conf:
+    # pick up a model from https://huggingface.co/models?filter=transformers
+    # most of models should work, but maybe some don't
+    # known to work: bert, gpt2, xlnet, roberta, mpnet, t5, bart
+    # xlnet currently works for single gpu only
+    model_name_or_path: "microsoft/deberta-large"
+    output_size: 256
+
+freeze_param:
+    - "frontend.upstream"
+    - "encoder"
+    - "postdecoder.model"
+
+seed: 2022
+log_interval: 100
+num_workers: 2
+batch_type: numel
+batch_bins: 6000000
+accum_grad: 2
+max_epoch: 55
+patience: none
+init: none
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+    lr: 0.002
+    weight_decay: 0.000001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wavlm_conv2d2.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wavlm_conv2d2.yaml
new file mode 100644
index 00000000000..e14cfbcd2b6
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wavlm_conv2d2.yaml
@@ -0,0 +1,88 @@
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 27
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_ratio_range:
+    - 0.
+    - 0.05
+    num_time_mask: 2
+
+encoder: conformer
+encoder_conf:
+    output_size: 256
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d2
+    normalize_before: true
+    macaron_style: true
+    rel_pos_type: latest
+    pos_enc_layer_type: rel_pos
+    selfattention_layer_type: rel_selfattn
+    activation_type: swish
+    use_cnn_module: true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
+
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream: wavlm_large  # Note: If the upstream is changed, please change the input_size in the preencoder.
+    download_dir: ./hub
+    multilayer_feature: true
+
+preencoder: linear
+preencoder_conf:
+    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+freeze_param:
+    - "frontend.upstream"
+
+seed: 2022
+log_interval: 100
+num_workers: 2
+batch_type: numel
+batch_bins: 6000000
+accum_grad: 2
+max_epoch: 70
+patience: none
+init: none
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+    lr: 0.002
+    weight_decay: 0.000001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr3e-3_warmup5k_wavlm_conv2d2.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr3e-3_warmup5k_wavlm_conv2d2.yaml
new file mode 100644
index 00000000000..5d5852133cc
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr3e-3_warmup5k_wavlm_conv2d2.yaml
@@ -0,0 +1,88 @@
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 27
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_ratio_range:
+    - 0.
+    - 0.05
+    num_time_mask: 2
+
+encoder: conformer
+encoder_conf:
+    output_size: 256
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d2
+    normalize_before: true
+    macaron_style: true
+    rel_pos_type: latest
+    pos_enc_layer_type: rel_pos
+    selfattention_layer_type: rel_selfattn
+    activation_type: swish
+    use_cnn_module: true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
+
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream: wavlm_large  # Note: If the upstream is changed, please change the input_size in the preencoder.
+    download_dir: ./hub
+    multilayer_feature: true
+
+preencoder: linear
+preencoder_conf:
+    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+freeze_param:
+    - "frontend.upstream"
+
+seed: 2022
+log_interval: 100
+num_workers: 2
+batch_type: numel
+batch_bins: 6000000
+accum_grad: 2
+max_epoch: 70
+patience: none
+init: none
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+    lr: 0.003
+    weight_decay: 0.000001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr5e-4_warmup5k_wavlm_conv2d2.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr5e-4_warmup5k_wavlm_conv2d2.yaml
new file mode 100644
index 00000000000..8cb1dfe8737
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr5e-4_warmup5k_wavlm_conv2d2.yaml
@@ -0,0 +1,88 @@
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 27
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_ratio_range:
+    - 0.
+    - 0.05
+    num_time_mask: 2
+
+encoder: conformer
+encoder_conf:
+    output_size: 256
+    attention_heads: 4
+    linear_units: 1024
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d2
+    normalize_before: true
+    macaron_style: true
+    rel_pos_type: latest
+    pos_enc_layer_type: rel_pos
+    selfattention_layer_type: rel_selfattn
+    activation_type: swish
+    use_cnn_module: true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
+
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream: wavlm_large  # Note: If the upstream is changed, please change the input_size in the preencoder.
+    download_dir: ./hub
+    multilayer_feature: true
+
+preencoder: linear
+preencoder_conf:
+    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+freeze_param:
+    - "frontend.upstream"
+
+seed: 2022
+log_interval: 100
+num_workers: 2
+batch_type: numel
+batch_bins: 6000000
+accum_grad: 2
+max_epoch: 70
+patience: none
+init: none
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+    lr: 0.0005
+    weight_decay: 0.000001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/db.sh b/egs2/slue-voxpopuli/asr1/db.sh
new file mode 120000
index 00000000000..50d86130898
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/db.sh
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/db.sh
\ No newline at end of file
diff --git a/egs2/slue-voxpopuli/asr1/local/data.sh b/egs2/slue-voxpopuli/asr1/local/data.sh
new file mode 100755
index 00000000000..99112c53e59
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/local/data.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+log() {
+    local fname=${BASH_SOURCE[1]##*/}
+    echo -e "$(date '+%Y-%m-%dT%H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
+}
+SECONDS=0
+
+
+stage=1
+stop_stage=100000
+log "$0 $*"
+. utils/parse_options.sh
+
+. ./db.sh
+. ./path.sh
+. ./cmd.sh
+
+if [ $# -ne 0 ]; then
+    log "Error: No positional arguments are required."
+    exit 2
+fi
+
+if [ -z "${VOXPOPULI}" ]; then
+    log "Fill the value of 'VOXPOPULI' of db.sh"
+    exit 1
+fi
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    if [ ! -e "${VOXPOPULI}/LICENSE.txt" ]; then
+	echo "stage 1: Download data to ${VOXPOPULI}"
+    else
+        log "stage 1: ${VOXPOPULI}/LICENSE.txt is already existing. Skip data downloading"
+    fi
+fi
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+    log "stage 2: Data Preparation"
+
+    mkdir -p data/{train,devel,test}
+    python3 local/data_prep_original_slue_format.py ${VOXPOPULI}
+    for x in test devel train; do
+        for f in text wav.scp utt2spk transcript; do
+            sort data/${x}/${f} -o data/${x}/${f}
+        done
+        utils/utt2spk_to_spk2utt.pl data/${x}/utt2spk > "data/${x}/spk2utt"
+        utils/validate_data_dir.sh --no-feats data/${x} || exit 1
+    done
+fi
+
+log "Successfully finished. [elapsed=${SECONDS}s]"
diff --git a/egs2/slue-voxpopuli/asr1/local/data_prep_original_slue_format.py b/egs2/slue-voxpopuli/asr1/local/data_prep_original_slue_format.py
new file mode 100644
index 00000000000..005da336b83
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/local/data_prep_original_slue_format.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+
+import os
+import pandas as pd
+import re
+import sys
+
+if len(sys.argv) != 2:
+    print("Usage: python data_prep.py [root]")
+    sys.exit(1)
+root = sys.argv[1]
+
+dir_dict = {
+    "train": "slue-voxpopuli_fine-tune.tsv",
+    "devel": "slue-voxpopuli_dev.tsv",
+    "test": "slue-voxpopuli_test_blind.tsv",
+}
+
+ontonotes_to_combined_label = {
+    "GPE": "PLACE",
+    "LOC": "PLACE",
+    "CARDINAL": "QUANT",
+    "MONEY": "QUANT",
+    "ORDINAL": "QUANT",
+    "PERCENT": "QUANT",
+    "QUANTITY": "QUANT",
+    "ORG": "ORG",
+    "DATE": "WHEN",
+    "TIME": "WHEN",
+    "NORP": "NORP",
+    "PERSON": "PERSON",
+    "LAW": "LAW",
+}
+
+
+missing_count = 0
+missing_ent = set()
+
+for x in dir_dict:
+    with open(os.path.join("data", x, "text"), "w") as text_f, open(
+        os.path.join("data", x, "wav.scp"), "w"
+    ) as wav_scp_f, open(os.path.join("data", x, "utt2spk"), "w") as utt2spk_f:
+
+        text_f.truncate()
+        wav_scp_f.truncate()
+        utt2spk_f.truncate()
+        transcript_df = pd.read_csv(os.path.join(root, dir_dict[x]), sep="\t")
+        # lines = sorted(transcript_df.values, key=lambda s: s[0])
+        for row in transcript_df.values:
+            uttid = row[3] + "_" + row[0]
+            speaker = row[3]
+            if x == "train":
+                wav = "fine-tune/" + row[0] + ".ogg"
+            elif x == "devel":
+                wav = "dev/" + row[0] + ".ogg"
+            else:
+                wav = "test/" + row[0] + ".ogg"
+
+            transcript = row[2].lower()
+            entities = []
+            if x != "test":  # blind test set
+                if str(row[6]) != "None":
+                    for slot in row[6].split("], "):
+                        ent_type = (
+                            slot.split(",")[0]
+                            .replace("[", "")
+                            .replace("]", "")
+                            .replace('"', "")
+                            .replace("'", "")
+                        )
+                        if ent_type in ontonotes_to_combined_label:
+                            ent_type = ontonotes_to_combined_label[ent_type]
+                        else:
+                            missing_count += 1
+                            missing_ent.add(ent_type)
+                            continue
+                        fill_start = int(
+                            slot.split(",")[1]
+                            .replace("[", "")
+                            .replace("]", "")
+                            .replace('"', "")
+                            .replace("'", "")
+                            .replace(" ", "")
+                        )
+                        fill_len = int(
+                            slot.split(",")[2]
+                            .replace("[", "")
+                            .replace("]", "")
+                            .replace('"', "")
+                            .replace("'", "")
+                            .replace(" ", "")
+                        )
+                        filler = transcript[fill_start : fill_start + fill_len]
+                        entities.append(
+                            {
+                                "type": ent_type,
+                                "filler": filler,
+                                "filler_start": fill_start,
+                                "filler_end": fill_start + fill_len,
+                            }
+                        )
+            new_transcript = transcript[:]
+            for entity in entities:
+                new_transcript = (
+                    new_transcript[: entity["filler_start"]]
+                    + entity["type"]
+                    + " FILL "
+                    + entity["filler"].lower()
+                    + " SEP "
+                    + new_transcript[entity["filler_end"] :]
+                )
+
+            words = "{}".format(new_transcript).replace("<unk>", "unknown")
+            words = re.sub(r"[\.;?!]", "", words)
+            words = re.sub(r"\s+", " ", words)
+
+            text_f.write("{} {}\n".format(uttid, words))
+            utt2spk_f.write("{} {}\n".format(uttid, speaker))
+            wav_scp_f.write(f"{uttid} sox {os.path.join(root,wav)} -t wav -r 16k - |\n")
+
+print("Missing Entities", missing_ent)
+print("Missing Count", missing_count)
diff --git a/egs2/slue-voxpopuli/asr1/local/data_prep_original_slue_format_transcript.py b/egs2/slue-voxpopuli/asr1/local/data_prep_original_slue_format_transcript.py
new file mode 100644
index 00000000000..622c830e4cf
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/local/data_prep_original_slue_format_transcript.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+
+import os
+import pandas as pd
+import re
+import sys
+
+if len(sys.argv) != 2:
+    print("Usage: python data_prep.py [root]")
+    sys.exit(1)
+root = sys.argv[1]
+
+dir_dict = {
+    "train": "slue-voxpopuli_fine-tune.tsv",
+    "devel": "slue-voxpopuli_dev.tsv",
+    "test": "slue-voxpopuli_test_blind.tsv",
+}
+
+ontonotes_to_combined_label = {
+    "GPE": "PLACE",
+    "LOC": "PLACE",
+    "CARDINAL": "QUANT",
+    "MONEY": "QUANT",
+    "ORDINAL": "QUANT",
+    "PERCENT": "QUANT",
+    "QUANTITY": "QUANT",
+    "ORG": "ORG",
+    "DATE": "WHEN",
+    "TIME": "WHEN",
+    "NORP": "NORP",
+    "PERSON": "PERSON",
+    "LAW": "LAW",
+}
+
+
+missing_count = 0
+missing_ent = set()
+
+for x in dir_dict:
+    with open(os.path.join("data", x, "text"), "w") as text_f, open(
+        os.path.join("data", x, "transcript"), "w"
+    ) as transcript_f, open(os.path.join("data", x, "wav.scp"), "w") as wav_scp_f, open(
+        os.path.join("data", x, "utt2spk"), "w"
+    ) as utt2spk_f:
+
+        text_f.truncate()
+        wav_scp_f.truncate()
+        utt2spk_f.truncate()
+        transcript_df = pd.read_csv(os.path.join(root, dir_dict[x]), sep="\t")
+        # lines = sorted(transcript_df.values, key=lambda s: s[0])
+        for row in transcript_df.values:
+            uttid = row[3] + "_" + row[0]
+            speaker = row[3]
+            if x == "train":
+                wav = "fine-tune/" + row[0] + ".ogg"
+            elif x == "devel":
+                wav = "dev/" + row[0] + ".ogg"
+            else:
+                wav = "test/" + row[0] + ".ogg"
+
+            transcript = row[2].lower()
+            entities = []
+            if x != "test":  # blind test set
+                if str(row[6]) != "None":
+                    for slot in row[6].split("], "):
+                        ent_type = (
+                            slot.split(",")[0]
+                            .replace("[", "")
+                            .replace("]", "")
+                            .replace('"', "")
+                            .replace("'", "")
+                        )
+                        if ent_type in ontonotes_to_combined_label:
+                            ent_type = ontonotes_to_combined_label[ent_type]
+                        else:
+                            missing_count += 1
+                            missing_ent.add(ent_type)
+                            continue
+                        fill_start = int(
+                            slot.split(",")[1]
+                            .replace("[", "")
+                            .replace("]", "")
+                            .replace('"', "")
+                            .replace("'", "")
+                            .replace(" ", "")
+                        )
+                        fill_len = int(
+                            slot.split(",")[2]
+                            .replace("[", "")
+                            .replace("]", "")
+                            .replace('"', "")
+                            .replace("'", "")
+                            .replace(" ", "")
+                        )
+                        filler = transcript[fill_start : fill_start + fill_len]
+                        entities.append(
+                            {
+                                "type": ent_type,
+                                "filler": filler,
+                                "filler_start": fill_start,
+                                "filler_end": fill_start + fill_len,
+                            }
+                        )
+            new_transcript = transcript[:]
+            for entity in entities:
+                new_transcript = (
+                    new_transcript[: entity["filler_start"]]
+                    + entity["type"]
+                    + " FILL "
+                    + entity["filler"].lower()
+                    + " SEP "
+                    + new_transcript[entity["filler_end"] :]
+                )
+
+            words = "{}".format(new_transcript).replace("<unk>", "unknown")
+            words = re.sub(r"[\.;?!]", "", words)
+            words = re.sub(r"\s+", " ", words)
+
+            text_f.write("{} {}\n".format(uttid, words))
+            words = "{}".format(transcript).replace("<unk>", "unknown")
+            words = re.sub(r"[\.;?!]", "", words)
+            words = re.sub(r"\s+", " ", words)
+            transcript_f.write("{} {}\n".format(uttid, words))
+            utt2spk_f.write("{} {}\n".format(uttid, speaker))
+            wav_scp_f.write(f"{uttid} sox {os.path.join(root,wav)} -t wav -r 16k - |\n")
+
+print("Missing Entities", missing_ent)
+print("Missing Count", missing_count)
diff --git a/egs2/slue-voxpopuli/asr1/local/data_prep_slue.py b/egs2/slue-voxpopuli/asr1/local/data_prep_slue.py
new file mode 100644
index 00000000000..89b42059e30
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/local/data_prep_slue.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+# Copyright 2021  Siddhant Arora
+#           2021  Carnegie Mellon University
+# Apache 2.0
+
+
+import os
+import re
+import sys
+import pandas as pd
+
+if len(sys.argv) != 2:
+    print("Usage: python data_prep.py [root]")
+    sys.exit(1)
+root = sys.argv[1]
+
+dir_dict = {
+    "train": "slue-voxceleb_fine-tune.tsv",
+    "devel": "slue-voxceleb_dev.tsv",
+    "test": "slue-voxceleb_test_blind.tsv",
+}
+
+for x in dir_dict:
+    with open(os.path.join("data", x, "text"), "w") as text_f, open(
+        os.path.join("data", x, "wav.scp"), "w"
+    ) as wav_scp_f, open(
+        os.path.join("data", x, "transcript"), "w"
+    ) as transcript_f, open(
+        os.path.join("data", x, "utt2spk"), "w"
+    ) as utt2spk_f:
+
+        text_f.truncate()
+        wav_scp_f.truncate()
+        utt2spk_f.truncate()
+        transcript_df = pd.read_csv(os.path.join(root, dir_dict[x]), sep="\t")
+        # lines = sorted(transcript_df.values, key=lambda s: s[0])
+        for row in transcript_df.values:
+            if x == "test":
+                speaker = row[1]
+                words = (
+                    "<blank>"  # Test set is blind, will have to submit to leaderboard
+                )
+            else:
+                if row[4] == "<mixed>":
+                    continue
+                print(x)
+                print(row)
+                words = (
+                    row[4].replace(" ", "_")
+                    + " "
+                    + row[1].encode("ascii", "ignore").decode()
+                )
+                print(words)
+                speaker = row[2]
+            if x == "train":
+                path = "fine-tune_raw/" + row[0] + ".flac"
+            elif x == "devel":
+                path = "dev_raw/" + row[0] + ".flac"
+            else:
+                path = "test_raw/" + row[0] + ".flac"
+            utt_id = row[0]
+            # print(utt_id + " " + words + "\n")
+            text_f.write(utt_id + " " + words + "\n")
+            wav_scp_f.write(utt_id + " " + root + "/" + path + "\n")
+            utt2spk_f.write(utt_id + " " + speaker + "\n")
diff --git a/egs2/slue-voxpopuli/asr1/local/eval_utils.py b/egs2/slue-voxpopuli/asr1/local/eval_utils.py
new file mode 100644
index 00000000000..fc5c3f9c473
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/local/eval_utils.py
@@ -0,0 +1,149 @@
+from typing import List
+from collections import defaultdict
+import numpy as np
+import editdistance
+
+
+def get_ner_scores(all_gt, all_predictions):
+    """
+    Evalutes per-label and overall (micro and macro) metrics of precision, recall, and fscore
+    Input:
+        all_gt/all_predictions:
+            List of list of tuples: (label, phrase, identifier)
+            Each list of tuples correspond to a sentence:
+                label: entity tag
+                phrase: entity phrase
+                tuple_identifier: identifier to differentiate repeating (label, phrase) pairs
+    Returns:
+        Dictionary of metrics
+    Example:
+        List of GT (label, phrase) pairs of a sentence: [(GPE, "eu"), (DATE, "today"), (GPE, "eu")]
+        all_gt: [(GPE, "eu", 0), (DATE, "today", 0), (GPE, "eu", 1)]
+    """
+    metrics = {}
+    stats = get_ner_stats(all_gt, all_predictions)
+    num_correct, num_gt, num_pred = 0, 0, 0
+    prec_lst, recall_lst, fscore_lst = [], [], []
+    for tag_name, tag_stats in stats.items():
+        precision, recall, fscore = get_metrics(
+            np.sum(tag_stats["tp"]),
+            np.sum(tag_stats["gt_cnt"]),
+            np.sum(tag_stats["pred_cnt"]),
+        )
+        _ = metrics.setdefault(tag_name, {})
+        metrics[tag_name]["precision"] = precision
+        metrics[tag_name]["recall"] = recall
+        metrics[tag_name]["fscore"] = fscore
+
+        num_correct += np.sum(tag_stats["tp"])
+        num_pred += np.sum(tag_stats["pred_cnt"])
+        num_gt += np.sum(tag_stats["gt_cnt"])
+
+        prec_lst.append(precision)
+        recall_lst.append(recall)
+        fscore_lst.append(fscore)
+
+    precision, recall, fscore = get_metrics(num_correct, num_gt, num_pred)
+    metrics["overall_micro"] = {}
+    metrics["overall_micro"]["precision"] = precision
+    metrics["overall_micro"]["recall"] = recall
+    metrics["overall_micro"]["fscore"] = fscore
+
+    metrics["overall_macro"] = {}
+    metrics["overall_macro"]["precision"] = np.mean(prec_lst)
+    metrics["overall_macro"]["recall"] = np.mean(recall_lst)
+    metrics["overall_macro"]["fscore"] = np.mean(fscore_lst)
+
+    return metrics
+
+
+def get_ner_stats(all_gt, all_predictions):
+    stats = {}
+    cnt = 0
+    for gt, pred in zip(all_gt, all_predictions):
+        entities_true = defaultdict(set)
+        entities_pred = defaultdict(set)
+        for type_name, entity_info1, entity_info2 in gt:
+            entities_true[type_name].add((entity_info1, entity_info2))
+        for type_name, entity_info1, entity_info2 in pred:
+            entities_pred[type_name].add((entity_info1, entity_info2))
+        target_names = sorted(set(entities_true.keys()) | set(entities_pred.keys()))
+        for tag_name in target_names:
+            _ = stats.setdefault(tag_name, {})
+            _ = stats[tag_name].setdefault("tp", [])
+            _ = stats[tag_name].setdefault("gt_cnt", [])
+            _ = stats[tag_name].setdefault("pred_cnt", [])
+            entities_true_type = entities_true.get(tag_name, set())
+            entities_pred_type = entities_pred.get(tag_name, set())
+            stats[tag_name]["tp"].append(len(entities_true_type & entities_pred_type))
+            stats[tag_name]["pred_cnt"].append(len(entities_pred_type))
+            stats[tag_name]["gt_cnt"].append(len(entities_true_type))
+    return stats
+
+
+def safe_divide(numerator, denominator):
+    numerator = np.array(numerator)
+    denominator = np.array(denominator)
+    mask = denominator == 0.0
+    denominator = denominator.copy()
+    denominator[mask] = 1  # avoid infs/nans
+    return numerator / denominator
+
+
+def ner_error_analysis(all_gt, all_predictions, gt_text):
+    """
+    Print out predictions and GT
+    all_gt: [GT] list of tuples of (label, phrase, identifier idx)
+    all_predictions: [hypothesis] list of tuples of (label, phrase, identifier idx)
+    gt_text: list of GT text sentences
+    """
+    analysis_examples_dct = {}
+    analysis_examples_dct["all"] = []
+    for idx, text in enumerate(gt_text):
+        if isinstance(text, list):
+            text = " ".join(text)
+        gt = all_gt[idx]
+        pred = all_predictions[idx]
+        entities_true = defaultdict(set)
+        entities_pred = defaultdict(set)
+        for type_name, entity_info1, entity_info2 in gt:
+            entities_true[type_name].add((entity_info1, entity_info2))
+        for type_name, entity_info1, entity_info2 in pred:
+            entities_pred[type_name].add((entity_info1, entity_info2))
+        target_names = sorted(set(entities_true.keys()) | set(entities_pred.keys()))
+        analysis_examples_dct["all"].append("\t".join([text, str(gt), str(pred)]))
+        for tag_name in target_names:
+            _ = analysis_examples_dct.setdefault(tag_name, [])
+            entities_true_type = entities_true.get(tag_name, set())
+            entities_pred_type = entities_pred.get(tag_name, set())
+            num_gt = len(entities_true_type)
+            num_correct = len(entities_true_type & entities_pred_type)
+
+            new_gt = [(item1, item2) for item1, item2, _ in gt]
+            new_pred = [(item1, item2) for item1, item2, _ in pred]
+            analysis_examples_dct[tag_name].append(
+                "\t".join([text, str(new_gt), str(new_pred)])
+            )
+
+    return analysis_examples_dct
+
+
+def get_metrics(num_correct, num_gt, num_pred):
+    precision = safe_divide([num_correct], [num_pred])
+    recall = safe_divide([num_correct], [num_gt])
+    fscore = safe_divide([2 * precision * recall], [(precision + recall)])
+    return precision[0], recall[0], fscore[0][0]
+
+
+def get_wer(refs: List[str], hyps: List[str]):
+    """
+    args:
+        refs (list of str): reference texts
+        hyps (list of str): hypothesis/prediction texts
+    """
+    n_words, n_errors = 0, 0
+    for ref, hyp in zip(refs, hyps):
+        ref, hyp = ref.split(), hyp.split()
+        n_words += len(ref)
+        n_errors += editdistance.eval(ref, hyp)
+    return safe_divide(n_errors, n_words)
diff --git a/egs2/slue-voxpopuli/asr1/local/f1_score.py b/egs2/slue-voxpopuli/asr1/local/f1_score.py
new file mode 100755
index 00000000000..4f45752a812
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/local/f1_score.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+
+# Copyright 2021  Siddhant Arora
+#           2021  Carnegie Mellon University
+# Apache 2.0
+
+
+import os
+import re
+import sys
+import pandas as pd
+import argparse
+from sklearn.metrics import f1_score
+from sklearn.metrics import classification_report
+
+
+def get_classification_result(hyp_file, ref_file):
+    hyp_lines = [line for line in hyp_file]
+    ref_lines = [line for line in ref_file]
+
+    error = 0
+    hyp_intent_arr = []
+    ref_intent_arr = []
+    for line_count in range(len(hyp_lines)):
+        hyp_intent = hyp_lines[line_count].split(" ")[0]
+        ref_intent = ref_lines[line_count].split(" ")[0]
+        hyp_intent_arr.append(hyp_intent)
+        ref_intent_arr.append(ref_intent)
+    print(classification_report(ref_intent_arr, hyp_intent_arr))
+    return f1_score(ref_intent_arr, hyp_intent_arr, average="macro")
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--exp_root", required=True, help="Directory to save experiments")
+parser.add_argument(
+    "--valid_folder",
+    default="inference_asr_model_valid.acc.ave_10best/devel/",
+    help="Directory inside exp_root containing inference on valid set",
+)
+parser.add_argument(
+    "--test_folder",
+    default="inference_asr_model_valid.acc.ave_10best/test/",
+    help="Directory inside exp_root containing inference on test set",
+)
+parser.add_argument(
+    "--utterance_test_folder",
+    default=None,
+    help="Directory inside exp_root containing inference on utterance test set",
+)
+
+args = parser.parse_args()
+
+exp_root = args.exp_root
+valid_inference_folder = args.valid_folder
+test_inference_folder = args.test_folder
+
+valid_hyp_file = open(
+    os.path.join(exp_root, valid_inference_folder + "score_wer/hyp.trn")
+)
+valid_ref_file = open(
+    os.path.join(exp_root, valid_inference_folder + "score_wer/ref.trn")
+)
+
+result = get_classification_result(valid_hyp_file, valid_ref_file)
+print("Valid Macro F1")
+print(result)
+
+test_hyp_file = open(
+    os.path.join(exp_root, test_inference_folder + "score_wer/hyp.trn")
+)
+test_ref_file = open(
+    os.path.join(exp_root, test_inference_folder + "score_wer/ref.trn")
+)
+
+result = get_classification_result(test_hyp_file, test_ref_file)
+print("Test Intent Macro F1")
+print(result)
+
+if args.utterance_test_folder is not None:
+    utt_test_inference_folder = args.utterance_test_folder
+    utt_test_hyp_file = open(
+        os.path.join(exp_root, utt_test_inference_folder + "score_wer/hyp.trn")
+    )
+    utt_test_ref_file = open(
+        os.path.join(exp_root, utt_test_inference_folder + "score_wer/ref.trn")
+    )
+    result = get_classification_result(utt_test_hyp_file, utt_test_ref_file)
+    print("Unseen Utterance Test Macro F1")
+    print(result)
diff --git a/egs2/slue-voxpopuli/asr1/local/generate_asr_files.py b/egs2/slue-voxpopuli/asr1/local/generate_asr_files.py
new file mode 100644
index 00000000000..dd8a4645410
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/local/generate_asr_files.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+
+# Copyright 2021  Siddhant Arora
+#           2021  Carnegie Mellon University
+# Apache 2.0
+
+
+import os
+import re
+import sys
+import pandas as pd
+import argparse
+
+
+def generate_asr_files(txt_file, transcript_file):
+    line_arr = [line for line in txt_file]
+    for line in line_arr:
+        if len(line.split("\t")) > 2:
+            print(line)
+            exit()
+        if len(line.split("\t")[0].split()) == 1:
+            text = "<blank>"
+        else:
+            text = line.split("\t")[0].split()[1].replace("▁", "")
+        for sub_word in line.split("\t")[0].split()[2:]:
+            if "▁" in sub_word:
+                text = text + " " + sub_word.replace("▁", "")
+            else:
+                text = text + sub_word
+        if len(text) == 0:
+            text = "<blank>"
+        wav_name = line.split("\t")[1]
+        transcript_file.write(text + "\t" + wav_name)
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--exp_root", required=True, help="Directory to save experiments")
+parser.add_argument(
+    "--valid_folder",
+    default="inference_asr_model_valid.acc.ave_10best/devel/",
+    help="Directory inside exp_root containing inference on valid set",
+)
+parser.add_argument(
+    "--test_folder",
+    default="inference_asr_model_valid.acc.ave_10best/test/",
+    help="Directory inside exp_root containing inference on test set",
+)
+parser.add_argument(
+    "--utterance_test_folder",
+    default=None,
+    help="Directory inside exp_root containing inference on utterance test set",
+)
+
+args = parser.parse_args()
+
+exp_root = args.exp_root
+valid_inference_folder = args.valid_folder
+test_inference_folder = args.test_folder
+
+valid_hyp_file = open(
+    os.path.join(exp_root, valid_inference_folder + "score_wer/hyp.trn")
+)
+valid_ref_file = open(
+    os.path.join(exp_root, valid_inference_folder + "score_wer/ref.trn")
+)
+
+valid_hyp_write_file = open(
+    os.path.join(exp_root, valid_inference_folder + "score_wer/hyp_asr.trn"), "w"
+)
+valid_ref_write_file = open(
+    os.path.join(exp_root, valid_inference_folder + "score_wer/ref_asr.trn"), "w"
+)
+
+generate_asr_files(valid_hyp_file, valid_hyp_write_file)
+
+generate_asr_files(valid_ref_file, valid_ref_write_file)
+
+
+test_hyp_file = open(
+    os.path.join(exp_root, test_inference_folder + "score_wer/hyp.trn")
+)
+test_ref_file = open(
+    os.path.join(exp_root, test_inference_folder + "score_wer/ref.trn")
+)
+test_hyp_write_file = open(
+    os.path.join(exp_root, test_inference_folder + "score_wer/hyp_asr.trn"), "w"
+)
+test_ref_write_file = open(
+    os.path.join(exp_root, test_inference_folder + "score_wer/ref_asr.trn"), "w"
+)
+
+generate_asr_files(test_hyp_file, test_hyp_write_file)
+
+generate_asr_files(test_ref_file, test_ref_write_file)
+
+
+if args.utterance_test_folder is not None:
+    utt_test_inference_folder = args.utterance_test_folder
+    utt_test_hyp_file = open(
+        os.path.join(exp_root, utt_test_inference_folder + "score_wer/hyp.trn")
+    )
+    utt_test_ref_file = open(
+        os.path.join(exp_root, utt_test_inference_folder + "score_wer/ref.trn")
+    )
+    utt_test_hyp_write_file = open(
+        os.path.join(exp_root, utt_test_inference_folder + "score_wer/hyp_asr.trn"), "w"
+    )
+    utt_test_ref_write_file = open(
+        os.path.join(exp_root, utt_test_inference_folder + "score_wer/ref_asr.trn"), "w"
+    )
+    generate_asr_files(utt_test_hyp_file, utt_test_hyp_write_file)
+
+    generate_asr_files(utt_test_ref_file, utt_test_ref_write_file)
diff --git a/egs2/slue-voxpopuli/asr1/local/path.sh b/egs2/slue-voxpopuli/asr1/local/path.sh
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/egs2/slue-voxpopuli/asr1/local/run_spm.sh b/egs2/slue-voxpopuli/asr1/local/run_spm.sh
new file mode 100755
index 00000000000..9710cf5601e
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/local/run_spm.sh
@@ -0,0 +1,38 @@
+# This script is called in data preparation step by local/data.sh
+# It takes the data prepared using token type word as input
+# It then trains a bpe model with "nbpe" number of tokens on the train transcript i.e. text after first word (intent)
+# It then encodes the transcript for train, valid and test using the trained bpe model 
+nbpe=1000 #try 100, 500, 1000
+bpemode=bpe #try unigram, bpe
+
+new_data=data_${bpemode}_${nbpe}
+dict=${new_data}/en_token_list/word/tokens.txt
+bpemodel=${new_data}/spm_train_${bpemode}${nbpe}
+
+cp -R data ${new_data}
+
+cut -d' ' -f2 data/train/text | sort | uniq > ${new_data}/intents.txt
+cut -d' ' -f3- data/train/text > ${new_data}/input.txt
+
+spm_train --input=${new_data}/input.txt \
+            --model_prefix=${bpemodel} \
+            --vocab_size=${nbpe} \
+            --character_coverage=1.0 \
+            --model_type=${bpemode} \
+            --model_prefix=${bpemodel} \
+            --input_sentence_size=100000000 \
+            --bos_id=-1 \
+            --eos_id=-1 \
+            --unk_id=0 
+
+for split in train devel test; do 
+    cut -d' ' -f-2 data/${split}/text > ${new_data}/tmp_${split}_utt
+    cut -d' ' -f3- data/${split}/text > ${new_data}/tmp_${split}_transcript
+    spm_encode --model=${bpemodel}.model --output_format=piece < ${new_data}/tmp_${split}_transcript > ${new_data}/new_${split}_transcript
+    paste -d' ' ${new_data}/tmp_${split}_utt ${new_data}/new_${split}_transcript > ${new_data}/${split}/text
+    rm ${new_data}/tmp_${split}_utt
+    rm ${new_data}/tmp_${split}_transcript
+    rm ${new_data}/new_${split}_transcript
+done
+
+#| awk '{print $0 " " NR+1}' >> ${dict}
diff --git a/egs2/slue-voxpopuli/asr1/local/score.py b/egs2/slue-voxpopuli/asr1/local/score.py
new file mode 120000
index 00000000000..32c1318cf74
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/local/score.py
@@ -0,0 +1 @@
+../../../TEMPLATE/asr1/pyscripts/utils/score_intent.py
\ No newline at end of file
diff --git a/egs2/slue-voxpopuli/asr1/local/score.sh b/egs2/slue-voxpopuli/asr1/local/score.sh
new file mode 100755
index 00000000000..39e6661e841
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/local/score.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Copyright 2021  Siddhant Arora
+#           2021  Carnegie Mellon University
+
+# # begin configuration section.
+# cmd=run.pl
+# stage=0
+# data=data/eval2000
+# #end configuration section.
+
+[ -f ./path.sh ] && . ./path.sh
+. parse_options.sh || exit 1;
+
+if [ $# -lt 1 ]; then
+  echo "Usage: local/score.sh <asr-exp-dir> <valid_inference_folder> <test_inference_folder>"
+  exit 1;
+fi
+. ./db.sh
+
+asr_expdir=$1
+
+if [ $# -gt 1 ]; then
+	valid_inference_folder=$2
+	test_inference_folder=$3
+	python local/score.py --exp_root ${asr_expdir} --valid_folder ${valid_inference_folder} --test_folder ${test_inference_folder}
+else
+	valid_inference_folder=decode_asr_asr_model_valid.acc.ave/devel
+	test_inference_folder=decode_asr_asr_model_valid.acc.ave/test
+	python local/score.py --exp_root ${asr_expdir}
+fi
+
+sclite \
+    -r "${asr_expdir}/${valid_inference_folder}/score_ter/ref_asr.trn" trn \
+    -h "${asr_expdir}/${valid_inference_folder}/score_ter/hyp_asr.trn" trn \
+    -i rm -o all stdout > "${asr_expdir}/${valid_inference_folder}/score_ter/result_asr.txt"
+echo "Write ASR result in ${asr_expdir}/${valid_inference_folder}/score_ter/result_asr.txt"
+grep -e Avg -e SPKR -m 2 "${asr_expdir}/${valid_inference_folder}/score_ter/result_asr.txt"
+
+if [ -d "${test_inference_folder}" ]; then
+	sclite \
+		-r "${asr_expdir}/${test_inference_folder}/score_ter/ref_asr.trn" trn \
+		-h "${asr_expdir}/${test_inference_folder}/score_ter/hyp_asr.trn" trn \
+		-i rm -o all stdout > "${asr_expdir}/${test_inference_folder}/score_ter/result_asr.txt"
+	echo "Write ASR result in ${asr_expdir}/${test_inference_folder}/score_ter/result_asr.txt"
+	grep -e Avg -e SPKR -m 2 "${asr_expdir}/${test_inference_folder}/score_ter/result_asr.txt"
+else
+	echo "[Warning] Skip ASR result on test set as it does not exist."
+fi
+
+exit 0
diff --git a/egs2/slue-voxpopuli/asr1/path.sh b/egs2/slue-voxpopuli/asr1/path.sh
new file mode 120000
index 00000000000..c9ac0a75bc6
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/path.sh
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/path.sh
\ No newline at end of file
diff --git a/egs2/slue-voxpopuli/asr1/pyscripts b/egs2/slue-voxpopuli/asr1/pyscripts
new file mode 120000
index 00000000000..ac68ad75b60
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/pyscripts
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/pyscripts
\ No newline at end of file
diff --git a/egs2/slue-voxpopuli/asr1/run.sh b/egs2/slue-voxpopuli/asr1/run.sh
new file mode 100755
index 00000000000..c3f8f2d9c90
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/run.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+train_set="train"
+valid_set="devel"
+test_sets="test devel"
+
+asr_config=conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wavlm_conv2d2.yaml
+inference_config=conf/decode_asr.yaml
+
+./asr.sh \
+    --lang en \
+    --ngpu 1 \
+    --use_lm false \
+    --token_type bpe \
+    --nbpe 1000 \
+    --bpe_nlsyms FILL,SEP,PLACE,QUANT,ORG,WHEN,NORP,PERSON,LAW \
+    --feats_type raw \
+    --audio_format "flac.ark" \
+    --speed_perturb_factors "0.9 1.0 1.1" \
+    --max_wav_duration 30 \
+    --feats_normalize utterance_mvn \
+    --asr_config "${asr_config}" \
+    --inference_config "${inference_config}" \
+    --train_set "${train_set}" \
+    --valid_set "${valid_set}" \
+    --test_sets "${test_sets}" \
+    --lm_train_text "data/${train_set}/text" \
+    --bpe_train_text "data/${train_set}/text" "$@"
diff --git a/egs2/slue-voxpopuli/asr1/scripts b/egs2/slue-voxpopuli/asr1/scripts
new file mode 120000
index 00000000000..b25829705dc
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/scripts
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/scripts
\ No newline at end of file
diff --git a/egs2/slue-voxpopuli/asr1/steps b/egs2/slue-voxpopuli/asr1/steps
new file mode 120000
index 00000000000..91f2d234e20
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/steps
@@ -0,0 +1 @@
+../../../tools/kaldi/egs/wsj/s5/steps
\ No newline at end of file
diff --git a/egs2/slue-voxpopuli/asr1/utils b/egs2/slue-voxpopuli/asr1/utils
new file mode 120000
index 00000000000..f49247da827
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/utils
@@ -0,0 +1 @@
+../../../tools/kaldi/egs/wsj/s5/utils
\ No newline at end of file

From c6a485fa5ab32752e00870eb3d9ba3b866e94952 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Tue, 12 Apr 2022 15:36:42 +0800
Subject: [PATCH 12/56] Fix unit test errors

---
 espnet2/enh/espnet_model.py                | 9 ++++++---
 espnet2/enh/loss/criterions/abs_loss.py    | 1 +
 espnet2/enh/loss/criterions/tf_domain.py   | 2 ++
 espnet2/enh/loss/criterions/time_domain.py | 1 +
 espnet2/enh/loss/wrappers/abs_wrapper.py   | 1 +
 test/espnet2/enh/test_espnet_model.py      | 4 ++--
 6 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/espnet2/enh/espnet_model.py b/espnet2/enh/espnet_model.py
index 04b74d1cfb3..1f87d8fd779 100644
--- a/espnet2/enh/espnet_model.py
+++ b/espnet2/enh/espnet_model.py
@@ -86,8 +86,8 @@ def forward(
         speech_ref = torch.stack(speech_ref, dim=1)
 
         if "noise_ref1" in kwargs:
-            # noise signal (optional, required when using
-            # frontend models with beamformering)
+            # noise signal (optional, required when using beamforming-based
+            # frontend models)
             noise_ref = [
                 kwargs["noise_ref{}".format(n + 1)] for n in range(self.num_noise_type)
             ]
@@ -162,7 +162,7 @@ def forward(
                 if criterion.compute_on_mask:
                     # compute loss on masks
                     if noise_ref is not None:
-                        noise_spec = self.encoder(noise_ref, speech_lengths)[0]
+                        noise_spec = self.encoder(noise_ref.sum(dim=1), speech_lengths)[0]
                     else:
                         noise_spec = None
                     tf_ref = criterion.create_mask_label(
@@ -184,6 +184,9 @@ def forward(
                     tf_pre = feature_pre
 
                 l, s, o = loss_wrapper(tf_ref, tf_pre, o)
+            else:
+                raise NotImplementedError("Unsupported loss type: %s" % str(criterion))
+
             loss += l * loss_wrapper.weight
             stats.update(s)
 
diff --git a/espnet2/enh/loss/criterions/abs_loss.py b/espnet2/enh/loss/criterions/abs_loss.py
index 11f8482fe05..c09119c9e07 100644
--- a/espnet2/enh/loss/criterions/abs_loss.py
+++ b/espnet2/enh/loss/criterions/abs_loss.py
@@ -8,6 +8,7 @@
 
 
 class AbsEnhLoss(torch.nn.Module, ABC):
+    """Base class for all Enhancement loss modules."""
 
     # the name will be the key that appears in the reporter
     @property
diff --git a/espnet2/enh/loss/criterions/tf_domain.py b/espnet2/enh/loss/criterions/tf_domain.py
index e3ca984a197..7342b33e112 100644
--- a/espnet2/enh/loss/criterions/tf_domain.py
+++ b/espnet2/enh/loss/criterions/tf_domain.py
@@ -88,10 +88,12 @@ def _create_mask_label(mix_spec, ref_spec, noise_spec=None, mask_type="IAM"):
             mask = new_complex_like(mix_spec, [mask_real, mask_imag])
         assert mask is not None, f"mask type {mask_type} not supported"
         mask_label.append(mask)
+    print([m.shape for m in mask_label])
     return mask_label
 
 
 class FrequencyDomainLoss(AbsEnhLoss, ABC):
+    """Base class for all frequence-domain Enhancement loss modules."""
 
     # The loss will be computed on mask or on spectrum
     @property
diff --git a/espnet2/enh/loss/criterions/time_domain.py b/espnet2/enh/loss/criterions/time_domain.py
index 87d75cfc483..8731c8f6d2e 100644
--- a/espnet2/enh/loss/criterions/time_domain.py
+++ b/espnet2/enh/loss/criterions/time_domain.py
@@ -7,6 +7,7 @@
 
 
 class TimeDomainLoss(AbsEnhLoss, ABC):
+    """Base class for all frequence-domain Enhancement loss modules."""
     pass
 
 
diff --git a/espnet2/enh/loss/wrappers/abs_wrapper.py b/espnet2/enh/loss/wrappers/abs_wrapper.py
index b0761272248..e48a2b7f869 100644
--- a/espnet2/enh/loss/wrappers/abs_wrapper.py
+++ b/espnet2/enh/loss/wrappers/abs_wrapper.py
@@ -8,6 +8,7 @@
 
 
 class AbsLossWrapper(torch.nn.Module, ABC):
+    """Base class for all Enhancement loss wrapper modules."""
 
     # The weight for the current loss in the multi-task learning.
     # The overall training target will be combined as:
diff --git a/test/espnet2/enh/test_espnet_model.py b/test/espnet2/enh/test_espnet_model.py
index f3a29f1757d..cf506b59179 100644
--- a/test/espnet2/enh/test_espnet_model.py
+++ b/test/espnet2/enh/test_espnet_model.py
@@ -203,13 +203,13 @@ def test_forward_with_beamformer_net(
         # `mask_type` has no effect when `loss_type` is not "mask..."
         return
     if not is_torch_1_9_plus and use_builtin_complex:
-        # builtin complex support is only available in PyTorch 1.8+
+        # builtin complex support is only well supported in PyTorch 1.9+
         return
 
     ch = 3
     inputs = random_speech[..., :ch].float()
     ilens = torch.LongTensor([16, 12])
-    speech_refs = [torch.randn(2, 16, ch).float() for spk in range(num_spk)]
+    speech_refs = [torch.randn(2, 16, ch, dtype=torch.float) for spk in range(num_spk)]
     noise_ref1 = torch.randn(2, 16, ch, dtype=torch.float)
     dereverb_ref1 = torch.randn(2, 16, ch, dtype=torch.float)
     encoder = STFTEncoder(

From a2352c4596087589b200bf4305c34cf3caa47526 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Tue, 12 Apr 2022 17:57:53 +0800
Subject: [PATCH 13/56] Add support for inference_config in ENH stage 7

---
 egs2/TEMPLATE/enh1/enh.sh              |  7 +-
 espnet2/bin/enh_inference.py           | 96 +++++++++++++++++++++++++-
 test/espnet2/bin/test_enh_inference.py | 24 +++++++
 3 files changed, 122 insertions(+), 5 deletions(-)

diff --git a/egs2/TEMPLATE/enh1/enh.sh b/egs2/TEMPLATE/enh1/enh.sh
index cb6e9e8503b..2c2ad7f3af4 100755
--- a/egs2/TEMPLATE/enh1/enh.sh
+++ b/egs2/TEMPLATE/enh1/enh.sh
@@ -78,6 +78,7 @@ download_model=
 # Evaluation related
 scoring_protocol="STOI SDR SAR SIR"
 ref_channel=0
+inference_enh_config= # Config for enhancement.
 score_with_asr=false
 asr_exp=""       # asr model for scoring WER
 lm_exp=""       # lm model for scoring WER
@@ -151,8 +152,9 @@ Options:
     --init_param    # pretrained model path and module name (default="${init_param}")
 
     # Enhancement related
-    --inference_args   # Arguments for enhancement in the inference stage (default="${inference_args}")
-    --inference_model  # Enhancement model path for inference (default="${inference_model}").
+    --inference_args       # Arguments for enhancement in the inference stage (default="${inference_args}")
+    --inference_model      # Enhancement model path for inference (default="${inference_model}").
+    --inference_enh_config # Configuration file for overwriting some model attributes during SE inference. (default="${inference_enh_config}")
 
     # Evaluation related
     --scoring_protocol    # Metrics to be used for scoring (default="${scoring_protocol}")
@@ -646,6 +648,7 @@ if ! "${skip_eval}"; then
                     --data_path_and_name_and_type "${_data}/${_scp},speech_mix,${_type}" \
                     --key_file "${_logdir}"/keys.JOB.scp \
                     --train_config "${enh_exp}"/config.yaml \
+                    ${inference_enh_config:+--inference_config "$inference_enh_config"} \
                     --model_file "${enh_exp}"/"${inference_model}" \
                     --output_dir "${_logdir}"/output.JOB \
                     ${_opts} ${inference_args}
diff --git a/espnet2/bin/enh_inference.py b/espnet2/bin/enh_inference.py
index f7c5ad797d2..86db63a4f4d 100755
--- a/espnet2/bin/enh_inference.py
+++ b/espnet2/bin/enh_inference.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import argparse
+from itertools import chain
 import logging
 from pathlib import Path
 import sys
@@ -15,6 +16,7 @@
 import torch
 from tqdm import trange
 from typeguard import check_argument_types
+import yaml
 
 from espnet.utils.cli_utils import get_commandline_args
 from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainMSE
@@ -24,6 +26,7 @@
 from espnet2.tasks.enh import EnhancementTask
 from espnet2.torch_utils.device_funcs import to_device
 from espnet2.torch_utils.set_all_random_seed import set_all_random_seed
+from espnet2.train.abs_espnet_model import AbsESPnetModel
 from espnet2.utils import config_argparse
 from espnet2.utils.types import str2bool
 from espnet2.utils.types import str2triple_str
@@ -33,6 +36,59 @@
 EPS = torch.finfo(torch.get_default_dtype()).eps
 
 
+def get_train_config(train_config, model_file=None):
+    if train_config is None:
+        assert model_file is not None, (
+            "The argument 'model_file' must be provided "
+            "if the argument 'train_config' is not specified."
+        )
+        train_config = Path(model_file).parent / "config.yaml"
+    else:
+        train_config = Path(train_config)
+    return train_config
+
+
+def recursive_dict_update(dict_org, dict_patch, verbose=False, log_prefix=""):
+    """Update `dict_org` with `dict_patch` in-place recursively."""
+    for key, value in dict_patch.items():
+        if key not in dict_org:
+            if verbose:
+                logging.info(
+                    "Overwriting config: {}{}: None -> {}".format(
+                        log_prefix, key, value
+                    )
+                )
+            dict_org[key] = value
+        elif isinstance(value, dict):
+            recursive_dict_update(
+                dict_org[key], value, verbose=verbose, log_prefix=f"{key}."
+            )
+        else:
+            if verbose and dict_org[key] != value:
+                logging.info(
+                    "Overwriting config: [{}{}]: {} -> {}".format(
+                        log_prefix, key, dict_org[key], value
+                    )
+                )
+            dict_org[key] = value
+
+
+def build_model_from_args_and_file(args, model_file, device):
+    model = EnhancementTask.build_model(args)
+    if not isinstance(model, AbsESPnetModel):
+        raise RuntimeError(
+            f"model must inherit {AbsESPnetModel.__name__}, but got {type(model)}"
+        )
+    model.to(device)
+    if model_file is not None:
+        if device == "cuda":
+            # NOTE(kamo): "cuda" for torch.load always indicates cuda:0
+            #   in PyTorch<=1.4
+            device = f"cuda:{torch.cuda.current_device()}"
+        model.load_state_dict(torch.load(model_file, map_location=device))
+    return model
+
+
 class SeparateSpeech:
     """SeparateSpeech class
 
@@ -49,6 +105,7 @@ def __init__(
         self,
         train_config: Union[Path, str] = None,
         model_file: Union[Path, str] = None,
+        inference_config: Union[Path, str] = None,
         segment_size: Optional[float] = None,
         hop_size: Optional[float] = None,
         normalize_segment_scale: bool = False,
@@ -61,9 +118,35 @@ def __init__(
         assert check_argument_types()
 
         # 1. Build Enh model
-        enh_model, enh_train_args = EnhancementTask.build_model_from_file(
-            train_config, model_file, device
-        )
+        if inference_config is None:
+            enh_model, enh_train_args = EnhancementTask.build_model_from_file(
+                train_config, model_file, device
+            )
+        else:
+            # Overwrite model attributes
+            train_config = get_train_config(train_config, model_file=model_file)
+            with train_config.open("r", encoding="utf-8") as f:
+                train_args = yaml.safe_load(f)
+
+            with Path(inference_config).open("r", encoding="utf-8") as f:
+                infer_args = yaml.safe_load(f)
+
+            supported_keys = list(
+                chain(*[[k, k + "_conf"] for k in ("encoder", "separator", "decoder")])
+            )
+            for k in infer_args.keys():
+                if k not in supported_keys:
+                    raise ValueError(
+                        "Only the following top-level keys are supported: %s"
+                        % ", ".join(supported_keys)
+                    )
+
+            recursive_dict_update(train_args, infer_args, verbose=True)
+            enh_train_args = argparse.Namespace(**train_args)
+            enh_model = build_model_from_args_and_file(
+                enh_train_args, model_file, device
+            )
+
         enh_model.to(dtype=getattr(torch, dtype)).eval()
 
         self.device = device
@@ -467,6 +550,13 @@ def get_parser():
         help="Pretrained model tag. If specify this option, train_config and "
         "model_file will be overwritten",
     )
+    group.add_argument(
+        "--inference_config",
+        type=str_or_none,
+        default=None,
+        help="Optional configuration file for overwriting enh model attributes "
+        "during inference",
+    )
 
     group = parser.add_argument_group("Data loading related")
     group.add_argument(
diff --git a/test/espnet2/bin/test_enh_inference.py b/test/espnet2/bin/test_enh_inference.py
index 5150e25823e..c4ec2a90285 100644
--- a/test/espnet2/bin/test_enh_inference.py
+++ b/test/espnet2/bin/test_enh_inference.py
@@ -8,6 +8,7 @@
 from espnet2.bin.enh_inference import main
 from espnet2.bin.enh_inference import SeparateSpeech
 from espnet2.tasks.enh import EnhancementTask
+from espnet2.utils.yaml_no_alias_safe_dump import yaml_no_alias_safe_dump
 
 
 def test_get_parser():
@@ -33,6 +34,20 @@ def config_file(tmp_path: Path):
     return tmp_path / "config.yaml"
 
 
+@pytest.fixture()
+def inference_config(tmp_path: Path):
+    # Write default configuration file
+    args = {
+        "encoder": "stft",
+        "encoder_conf": {"n_fft": 64, "hop_length": 32},
+        "decoder": "stft",
+        "decoder_conf": {"n_fft": 64, "hop_length": 32},
+    }
+    with open(tmp_path / "inference.yaml", "w") as f:
+        yaml_no_alias_safe_dump(args, f, indent=4, sort_keys=False)
+    return tmp_path / "inference.yaml"
+
+
 @pytest.mark.execution_timeout(5)
 @pytest.mark.parametrize("batch_size", [1, 2])
 @pytest.mark.parametrize(
@@ -50,3 +65,12 @@ def test_SeparateSpeech(
     )
     wav = torch.rand(batch_size, input_size)
     separate_speech(wav, fs=8000)
+
+
+@pytest.mark.execution_timeout(5)
+def test_SeparateSpeech_with_inference_config(config_file, inference_config):
+    separate_speech = SeparateSpeech(
+        train_config=config_file, inference_config=inference_config
+    )
+    wav = torch.rand(2, 16000)
+    separate_speech(wav, fs=8000)

From 3f5731e2e095f5ff8ca1e7de17af6d8c10c5e22e Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Tue, 12 Apr 2022 17:59:10 +0800
Subject: [PATCH 14/56] Apply black

---
 espnet2/enh/espnet_model.py                | 2 +-
 espnet2/enh/loss/criterions/time_domain.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/espnet2/enh/espnet_model.py b/espnet2/enh/espnet_model.py
index 1f87d8fd779..c6974e4ee28 100644
--- a/espnet2/enh/espnet_model.py
+++ b/espnet2/enh/espnet_model.py
@@ -162,7 +162,7 @@ def forward(
                 if criterion.compute_on_mask:
                     # compute loss on masks
                     if noise_ref is not None:
-                        noise_spec = self.encoder(noise_ref.sum(dim=1), speech_lengths)[0]
+                        noise_spec = self.encoder(noise_ref.sum(1), speech_lengths)[0]
                     else:
                         noise_spec = None
                     tf_ref = criterion.create_mask_label(
diff --git a/espnet2/enh/loss/criterions/time_domain.py b/espnet2/enh/loss/criterions/time_domain.py
index 8731c8f6d2e..c538c9c0bbf 100644
--- a/espnet2/enh/loss/criterions/time_domain.py
+++ b/espnet2/enh/loss/criterions/time_domain.py
@@ -8,6 +8,7 @@
 
 class TimeDomainLoss(AbsEnhLoss, ABC):
     """Base class for all frequence-domain Enhancement loss modules."""
+
     pass
 
 

From 93e775b11fc79df1967e00b6e3a36e4680f01081 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Tue, 12 Apr 2022 18:46:08 +0800
Subject: [PATCH 15/56] Add inference_tag to enh.sh for stage 7+

---
 egs2/TEMPLATE/enh1/enh.sh | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/egs2/TEMPLATE/enh1/enh.sh b/egs2/TEMPLATE/enh1/enh.sh
index 2c2ad7f3af4..ff9553ac118 100755
--- a/egs2/TEMPLATE/enh1/enh.sh
+++ b/egs2/TEMPLATE/enh1/enh.sh
@@ -78,6 +78,7 @@ download_model=
 # Evaluation related
 scoring_protocol="STOI SDR SAR SIR"
 ref_channel=0
+inference_tag=  # Prefix to the result dir for decoding.
 inference_enh_config= # Config for enhancement.
 score_with_asr=false
 asr_exp=""       # asr model for scoring WER
@@ -249,6 +250,14 @@ if [ -n "${speed_perturb_factors}" ]; then
   enh_exp="${enh_exp}_sp"
 fi
 
+if [ -z "${inference_tag}" ]; then
+    if [ -n "${inference_enh_config}" ]; then
+        inference_tag="$(basename "${inference_enh_config}" .yaml)"
+    else
+        inference_tag=enhanced
+    fi
+fi
+
 # ========================== Main stages start from here. ==========================
 
 if ! "${skip_data_prep}"; then
@@ -616,7 +625,7 @@ if ! "${skip_eval}"; then
 
         for dset in "${valid_set}" ${test_sets}; do
             _data="${data_feats}/${dset}"
-            _dir="${enh_exp}/enhanced_${dset}"
+            _dir="${enh_exp}/${inference_tag}_${dset}"
             _logdir="${_dir}/logdir"
             mkdir -p "${_logdir}"
 
@@ -689,7 +698,7 @@ if ! "${skip_eval}"; then
                 if "${score_obs}"; then
                     _dir="${data_feats}/${dset}/scoring"
                 else
-                    _dir="${enh_exp}/enhanced_${dset}/scoring"
+                    _dir="${enh_exp}/${inference_tag}_${dset}/scoring"
                 fi
 
                 _logdir="${_dir}/logdir"
@@ -716,7 +725,7 @@ if ! "${skip_eval}"; then
                         # To compute the score of observation, input original wav.scp
                         _inf_scp+="--inf_scp ${data_feats}/${dset}/wav.scp "
                     else
-                        _inf_scp+="--inf_scp ${enh_exp}/enhanced_${dset}/spk${spk}.scp "
+                        _inf_scp+="--inf_scp ${enh_exp}/${inference_tag}_${dset}/spk${spk}.scp "
                     fi
                 done
 
@@ -752,7 +761,7 @@ if ! "${skip_eval}"; then
             ./scripts/utils/show_enh_score.sh "${_dir}/../.." > "${_dir}/../../RESULTS.md"
         done
         log "Evaluation result for observation: ${data_feats}/RESULTS.md"
-        log "Evaluation result for enhancement: ${enh_exp}/enhanced/RESULTS.md"
+        log "Evaluation result for enhancement: ${enh_exp}/RESULTS.md"
 
     fi
 else
@@ -811,7 +820,7 @@ if "${score_with_asr}"; then
                         # Using same wav.scp for all speakers
                         cp "${_data}/wav.scp" "${_ddir}/wav.scp"
                     else
-                        cp "${enh_exp}/enhanced_${dset}/scoring/wav_spk${spk}" "${_ddir}/wav.scp"
+                        cp "${enh_exp}/${inference_tag}_${dset}/scoring/wav_spk${spk}" "${_ddir}/wav.scp"
                     fi
                     cp data/${dset}/text_spk${spk} ${_ddir}/text
                     cp ${_data}/{spk2utt,utt2spk,utt2num_samples,feats_type} ${_ddir}

From db08dfb95eb1182cf793d4441a42abe6caf2bbf3 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Tue, 12 Apr 2022 20:49:37 +0800
Subject: [PATCH 16/56] Fix unit test errors and remove debug code

---
 espnet2/bin/enh_inference.py             | 2 ++
 espnet2/enh/loss/criterions/tf_domain.py | 1 -
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/espnet2/bin/enh_inference.py b/espnet2/bin/enh_inference.py
index 86db63a4f4d..19c5e30e4c0 100755
--- a/espnet2/bin/enh_inference.py
+++ b/espnet2/bin/enh_inference.py
@@ -388,6 +388,7 @@ def inference(
     train_config: Optional[str],
     model_file: Optional[str],
     model_tag: Optional[str],
+    inference_config: Optional[str],
     allow_variable_data_keys: bool,
     segment_size: Optional[float],
     hop_size: Optional[float],
@@ -419,6 +420,7 @@ def inference(
     separate_speech_kwargs = dict(
         train_config=train_config,
         model_file=model_file,
+        inference_config=inference_config,
         segment_size=segment_size,
         hop_size=hop_size,
         normalize_segment_scale=normalize_segment_scale,
diff --git a/espnet2/enh/loss/criterions/tf_domain.py b/espnet2/enh/loss/criterions/tf_domain.py
index 7342b33e112..29a44c729b7 100644
--- a/espnet2/enh/loss/criterions/tf_domain.py
+++ b/espnet2/enh/loss/criterions/tf_domain.py
@@ -88,7 +88,6 @@ def _create_mask_label(mix_spec, ref_spec, noise_spec=None, mask_type="IAM"):
             mask = new_complex_like(mix_spec, [mask_real, mask_imag])
         assert mask is not None, f"mask type {mask_type} not supported"
         mask_label.append(mask)
-    print([m.shape for m in mask_label])
     return mask_label
 
 

From fc45f3f9d95ab86c931763e9692e15726b890188 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Tue, 12 Apr 2022 22:14:36 +0800
Subject: [PATCH 17/56] Improve unit test coverage

---
 espnet2/bin/enh_inference.py                  |  2 +-
 test/espnet2/bin/test_enh_inference.py        | 40 +++++++++++++++
 .../enh/loss/criterions/test_tf_domain.py     | 23 +++++++++
 .../enh/loss/criterions/test_time_domain.py   | 27 ++++++++--
 test/espnet2/enh/test_espnet_model.py         | 50 ++++++++++++++++++-
 5 files changed, 137 insertions(+), 5 deletions(-)

diff --git a/espnet2/bin/enh_inference.py b/espnet2/bin/enh_inference.py
index 19c5e30e4c0..eb51c4c1f86 100755
--- a/espnet2/bin/enh_inference.py
+++ b/espnet2/bin/enh_inference.py
@@ -54,7 +54,7 @@ def recursive_dict_update(dict_org, dict_patch, verbose=False, log_prefix=""):
         if key not in dict_org:
             if verbose:
                 logging.info(
-                    "Overwriting config: {}{}: None -> {}".format(
+                    "Overwriting config: [{}{}]: None -> {}".format(
                         log_prefix, key, value
                     )
                 )
diff --git a/test/espnet2/bin/test_enh_inference.py b/test/espnet2/bin/test_enh_inference.py
index c4ec2a90285..f9cf0920d86 100644
--- a/test/espnet2/bin/test_enh_inference.py
+++ b/test/espnet2/bin/test_enh_inference.py
@@ -3,11 +3,14 @@
 
 import pytest
 import torch
+import yaml
 
 from espnet2.bin.enh_inference import get_parser
 from espnet2.bin.enh_inference import main
 from espnet2.bin.enh_inference import SeparateSpeech
+from espnet2.enh.encoder.stft_encoder import STFTEncoder
 from espnet2.tasks.enh import EnhancementTask
+from espnet2.utils.get_default_kwargs import get_default_kwargs
 from espnet2.utils.yaml_no_alias_safe_dump import yaml_no_alias_safe_dump
 
 
@@ -31,6 +34,16 @@ def config_file(tmp_path: Path):
             str(tmp_path),
         ]
     )
+
+    with open(tmp_path / "config.yaml", "r") as f:
+        args = yaml.safe_load(f)
+
+    if args["encoder"] == "stft" and len(args["encoder_conf"]) == 0:
+        args["encoder_conf"] = get_default_kwargs(STFTEncoder)
+
+    with open(tmp_path / "config.yaml", "w") as f:
+        yaml_no_alias_safe_dump(args, f, indent=4, sort_keys=False)
+
     return tmp_path / "config.yaml"
 
 
@@ -48,6 +61,19 @@ def inference_config(tmp_path: Path):
     return tmp_path / "inference.yaml"
 
 
+@pytest.fixture()
+def invalid_inference_config(tmp_path: Path):
+    # Write default configuration file
+    args = {
+        "encoder": "stft",
+        "encoder_conf": {"n_fft": 64, "hop_length": 32},
+        "xxx": "invalid",
+    }
+    with open(tmp_path / "invalid_inference.yaml", "w") as f:
+        yaml_no_alias_safe_dump(args, f, indent=4, sort_keys=False)
+    return tmp_path / "invalid_inference.yaml"
+
+
 @pytest.mark.execution_timeout(5)
 @pytest.mark.parametrize("batch_size", [1, 2])
 @pytest.mark.parametrize(
@@ -74,3 +100,17 @@ def test_SeparateSpeech_with_inference_config(config_file, inference_config):
     )
     wav = torch.rand(2, 16000)
     separate_speech(wav, fs=8000)
+
+
+def test_SeparateSpeech_invalid_inference_config(
+    inference_config, invalid_inference_config
+):
+    with pytest.raises(AssertionError):
+        separate_speech = SeparateSpeech(
+            train_config=None, model_file=None, inference_config=inference_config
+        )
+
+    with pytest.raises(AssertionError):
+        separate_speech = SeparateSpeech(
+            train_config=None, inference_config=invalid_inference_config
+        )
diff --git a/test/espnet2/enh/loss/criterions/test_tf_domain.py b/test/espnet2/enh/loss/criterions/test_tf_domain.py
index d188b13f7bf..ed04806eb2a 100644
--- a/test/espnet2/enh/loss/criterions/test_tf_domain.py
+++ b/test/espnet2/enh/loss/criterions/test_tf_domain.py
@@ -40,6 +40,7 @@ def test_tf_domain_criterion_forward(
         loss = criterion(ref_spec[0], inf_spec[0])
 
     assert loss.shape == (batch,)
+    stats = {criterion.name: loss.detach()}
 
 
 @pytest.mark.parametrize("input_ch", [1, 2])
@@ -55,3 +56,25 @@ def test_tf_coh_criterion_forward(input_ch):
 
     loss = criterion(ref_spec, inf_spec)
     assert loss.shape == (batch,)
+    stats = {criterion.name: loss.detach()}
+
+
+@pytest.mark.parametrize("input_ch", [1, 2])
+def test_tf_coh_criterion_invalid_forward(input_ch):
+
+    criterion = FrequencyDomainAbsCoherence()
+    complex_wrapper = torch.complex if is_torch_1_9_plus else ComplexTensor
+
+    batch = 2
+    shape = (batch, 10, 200) if input_ch == 1 else (batch, 10, input_ch, 200)
+    inf_spec = complex_wrapper(torch.rand(*shape), torch.rand(*shape))
+    ref_spec = complex_wrapper(torch.rand(*shape), torch.rand(*shape))
+
+    with pytest.raises(ValueError):
+        loss = criterion(ref_spec.real, inf_spec)
+
+    with pytest.raises(ValueError):
+        if input_ch == 1:
+            loss = criterion(ref_spec[0], inf_spec[0])
+        else:
+            loss = criterion(ref_spec[0, 0], inf_spec[0, 0])
diff --git a/test/espnet2/enh/loss/criterions/test_time_domain.py b/test/espnet2/enh/loss/criterions/test_time_domain.py
index 53f50c88cf6..339143d7b0a 100644
--- a/test/espnet2/enh/loss/criterions/test_time_domain.py
+++ b/test/espnet2/enh/loss/criterions/test_time_domain.py
@@ -8,9 +8,7 @@
 from espnet2.enh.loss.criterions.time_domain import TimeDomainMSE
 
 
-@pytest.mark.parametrize(
-    "criterion_class", [CISDRLoss, SISNRLoss, SNRLoss, TimeDomainL1, TimeDomainMSE]
-)
+@pytest.mark.parametrize("criterion_class", [CISDRLoss, SISNRLoss, SNRLoss])
 def test_tf_domain_criterion_forward(criterion_class):
 
     criterion = criterion_class()
@@ -21,3 +19,26 @@ def test_tf_domain_criterion_forward(criterion_class):
 
     loss = criterion(ref, inf)
     assert loss.shape == (batch,)
+    stats = {criterion.name: loss.detach()}
+
+
+@pytest.mark.parametrize("criterion_class", [TimeDomainL1, TimeDomainMSE])
+@pytest.mark.parametrize("input_ch", [1, 2])
+def test_tf_domain_l1_l2_forward(criterion_class, input_ch):
+
+    criterion = criterion_class()
+
+    batch = 2
+    shape = (batch, 200) if input_ch == 1 else (batch, 200, input_ch)
+    inf = torch.rand(*shape)
+    ref = torch.rand(*shape)
+
+    loss = criterion(ref, inf)
+    assert loss.shape == (batch,)
+    stats = {criterion.name: loss.detach()}
+
+    with pytest.raises(ValueError):
+        if input_ch == 1:
+            loss = criterion(ref[..., None, None], inf[..., None, None])
+        else:
+            loss = criterion(ref[..., None], inf[..., None])
diff --git a/test/espnet2/enh/test_espnet_model.py b/test/espnet2/enh/test_espnet_model.py
index cf506b59179..c329105ecb4 100644
--- a/test/espnet2/enh/test_espnet_model.py
+++ b/test/espnet2/enh/test_espnet_model.py
@@ -209,7 +209,7 @@ def test_forward_with_beamformer_net(
     ch = 3
     inputs = random_speech[..., :ch].float()
     ilens = torch.LongTensor([16, 12])
-    speech_refs = [torch.randn(2, 16, ch, dtype=torch.float) for spk in range(num_spk)]
+    speech_refs = [torch.randn(2, 16, dtype=torch.float) for spk in range(num_spk)]
     noise_ref1 = torch.randn(2, 16, ch, dtype=torch.float)
     dereverb_ref1 = torch.randn(2, 16, ch, dtype=torch.float)
     encoder = STFTEncoder(
@@ -257,3 +257,51 @@ def test_forward_with_beamformer_net(
         "dereverb_ref1": dereverb_ref1,
     }
     loss, stats, weight = enh_model(**kwargs)
+
+
+@pytest.mark.parametrize("mask_type", ["IBM", "IRM"])
+@pytest.mark.parametrize("num_spk", [1, 2, 3])
+def test_invalid_case_for_masks(mask_type, num_spk):
+    ch = 3
+    inputs = random_speech[..., :ch].float()
+    ilens = torch.LongTensor([16, 12])
+    speech_refs = [torch.randn(2, 16, ch, dtype=torch.float) for spk in range(num_spk)]
+
+    encoder = STFTEncoder(n_fft=8, hop_length=2, use_builtin_complex=False)
+    decoder = STFTDecoder(n_fft=8, hop_length=2)
+
+    beamformer = NeuralBeamformer(
+        input_dim=5,
+        loss_type="mask_mse",
+        num_spk=num_spk,
+        use_wpe=True,
+        wlayers=2,
+        wunits=2,
+        wprojs=2,
+        use_dnn_mask_for_wpe=True,
+        multi_source_wpe=True,
+        use_beamformer=True,
+        blayers=2,
+        bunits=2,
+        bprojs=2,
+        badim=2,
+        ref_channel=0,
+        use_noise_mask=False,
+        beamformer_type="mvdr_souden",
+    )
+    enh_model = ESPnetEnhancementModel(
+        encoder=encoder,
+        decoder=decoder,
+        separator=beamformer,
+        loss_type="mask_mse",
+        mask_type=mask_type,
+        loss_wrappers=[pit_wrapper],
+    )
+
+    kwargs = {
+        "speech_mix": inputs,
+        "speech_mix_lengths": ilens,
+        **{"speech_ref{}".format(i + 1): speech_refs[i] for i in range(num_spk)},
+    }
+    with pytest.raises(ValueError):
+        loss, stats, weight = enh_model(**kwargs)

From 56d5040136c0e0c7599c84b4bc55497237f5eae8 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Tue, 12 Apr 2022 23:00:01 +0800
Subject: [PATCH 18/56] Fix python style issues

---
 test/espnet2/bin/test_enh_inference.py               |  6 ++----
 test/espnet2/enh/loss/criterions/test_tf_domain.py   | 12 +++++-------
 test/espnet2/enh/loss/criterions/test_time_domain.py |  6 ++----
 3 files changed, 9 insertions(+), 15 deletions(-)

diff --git a/test/espnet2/bin/test_enh_inference.py b/test/espnet2/bin/test_enh_inference.py
index f9cf0920d86..2274259531c 100644
--- a/test/espnet2/bin/test_enh_inference.py
+++ b/test/espnet2/bin/test_enh_inference.py
@@ -106,11 +106,9 @@ def test_SeparateSpeech_invalid_inference_config(
     inference_config, invalid_inference_config
 ):
     with pytest.raises(AssertionError):
-        separate_speech = SeparateSpeech(
+        SeparateSpeech(
             train_config=None, model_file=None, inference_config=inference_config
         )
 
     with pytest.raises(AssertionError):
-        separate_speech = SeparateSpeech(
-            train_config=None, inference_config=invalid_inference_config
-        )
+        SeparateSpeech(train_config=None, inference_config=invalid_inference_config)
diff --git a/test/espnet2/enh/loss/criterions/test_tf_domain.py b/test/espnet2/enh/loss/criterions/test_tf_domain.py
index ed04806eb2a..1a56039b475 100644
--- a/test/espnet2/enh/loss/criterions/test_tf_domain.py
+++ b/test/espnet2/enh/loss/criterions/test_tf_domain.py
@@ -39,8 +39,7 @@ def test_tf_domain_criterion_forward(
         inf_spec = [complex_wrapper(torch.rand(*shape), torch.rand(*shape))]
         loss = criterion(ref_spec[0], inf_spec[0])
 
-    assert loss.shape == (batch,)
-    stats = {criterion.name: loss.detach()}
+    assert loss.shape == (batch,), "Invlid loss shape with " + criterion.name
 
 
 @pytest.mark.parametrize("input_ch", [1, 2])
@@ -55,8 +54,7 @@ def test_tf_coh_criterion_forward(input_ch):
     ref_spec = complex_wrapper(torch.rand(*shape), torch.rand(*shape))
 
     loss = criterion(ref_spec, inf_spec)
-    assert loss.shape == (batch,)
-    stats = {criterion.name: loss.detach()}
+    assert loss.shape == (batch,), "Invlid loss shape with " + criterion.name
 
 
 @pytest.mark.parametrize("input_ch", [1, 2])
@@ -71,10 +69,10 @@ def test_tf_coh_criterion_invalid_forward(input_ch):
     ref_spec = complex_wrapper(torch.rand(*shape), torch.rand(*shape))
 
     with pytest.raises(ValueError):
-        loss = criterion(ref_spec.real, inf_spec)
+        criterion(ref_spec.real, inf_spec)
 
     with pytest.raises(ValueError):
         if input_ch == 1:
-            loss = criterion(ref_spec[0], inf_spec[0])
+            criterion(ref_spec[0], inf_spec[0])
         else:
-            loss = criterion(ref_spec[0, 0], inf_spec[0, 0])
+            criterion(ref_spec[0, 0], inf_spec[0, 0])
diff --git a/test/espnet2/enh/loss/criterions/test_time_domain.py b/test/espnet2/enh/loss/criterions/test_time_domain.py
index 339143d7b0a..34aad0f1440 100644
--- a/test/espnet2/enh/loss/criterions/test_time_domain.py
+++ b/test/espnet2/enh/loss/criterions/test_time_domain.py
@@ -18,8 +18,7 @@ def test_tf_domain_criterion_forward(criterion_class):
     ref = torch.rand(batch, 2000)
 
     loss = criterion(ref, inf)
-    assert loss.shape == (batch,)
-    stats = {criterion.name: loss.detach()}
+    assert loss.shape == (batch,), "Invlid loss shape with " + criterion.name
 
 
 @pytest.mark.parametrize("criterion_class", [TimeDomainL1, TimeDomainMSE])
@@ -34,8 +33,7 @@ def test_tf_domain_l1_l2_forward(criterion_class, input_ch):
     ref = torch.rand(*shape)
 
     loss = criterion(ref, inf)
-    assert loss.shape == (batch,)
-    stats = {criterion.name: loss.detach()}
+    assert loss.shape == (batch,), "Invlid loss shape with " + criterion.name
 
     with pytest.raises(ValueError):
         if input_ch == 1:

From a6cdbd1d4b190453bb689f56df2b6cb0e133c594 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Wed, 13 Apr 2022 00:12:07 +0800
Subject: [PATCH 19/56] Fix comment

---
 egs2/TEMPLATE/enh1/enh.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/egs2/TEMPLATE/enh1/enh.sh b/egs2/TEMPLATE/enh1/enh.sh
index 0afc31881e7..f64051a941c 100755
--- a/egs2/TEMPLATE/enh1/enh.sh
+++ b/egs2/TEMPLATE/enh1/enh.sh
@@ -78,7 +78,7 @@ download_model=
 # Evaluation related
 scoring_protocol="STOI SDR SAR SIR SI_SNR"
 ref_channel=0
-inference_tag=  # Prefix to the result dir for decoding.
+inference_tag=  # Prefix to the result dir for ENH inference.
 inference_enh_config= # Config for enhancement.
 score_with_asr=false
 asr_exp=""       # asr model for scoring WER

From 9bc1b5eede14e4d4c918454119d8d69f04064493 Mon Sep 17 00:00:00 2001
From: nateanl <nizhaoheng@gmail.com>
Date: Wed, 16 Mar 2022 13:25:26 +0000
Subject: [PATCH 20/56] add svoice recipe

---
 .../enh1/conf/tuning/train_enh_svoice.yaml    |  51 +++++
 espnet2/enh/layers/dpmulcat.py                | 158 ++++++++++++++++
 .../loss/wrappers/multilayer_pit_solver.py    |  36 ++++
 espnet2/enh/separator/svoice_separator.py     | 179 ++++++++++++++++++
 espnet2/tasks/enh.py                          |   5 +-
 5 files changed, 428 insertions(+), 1 deletion(-)
 create mode 100644 egs2/wsj0_2mix/enh1/conf/tuning/train_enh_svoice.yaml
 create mode 100644 espnet2/enh/layers/dpmulcat.py
 create mode 100644 espnet2/enh/loss/wrappers/multilayer_pit_solver.py
 create mode 100644 espnet2/enh/separator/svoice_separator.py

diff --git a/egs2/wsj0_2mix/enh1/conf/tuning/train_enh_svoice.yaml b/egs2/wsj0_2mix/enh1/conf/tuning/train_enh_svoice.yaml
new file mode 100644
index 00000000000..8ee8dc1d529
--- /dev/null
+++ b/egs2/wsj0_2mix/enh1/conf/tuning/train_enh_svoice.yaml
@@ -0,0 +1,51 @@
+optim: adam
+init: xavier_uniform
+max_epoch: 150
+batch_type: folded
+batch_size: 8
+iterator_type: chunk
+chunk_length: 16000
+num_workers: 4
+optim_conf:
+    lr: 1.0e-03
+    eps: 1.0e-08
+    weight_decay: 0
+patience: 20
+val_scheduler_criterion:
+- valid
+- loss
+best_model_criterion:
+-   - valid
+    - si_snr
+    - max
+-   - valid
+    - loss
+    - min
+keep_nbest_models: 1
+scheduler: reducelronplateau
+scheduler_conf:
+    mode: min
+    factor: 0.7
+    patience: 1
+
+encoder: same
+decoder: same
+separator: svoice
+separator_conf:
+    enc_dim: 128
+    kernel_size: 8
+    hidden_size: 128
+    num_spk: 2
+    num_layers: 6
+    segment_size: 128
+    input_normalize: False
+
+criterions: 
+  # The first criterion
+  - name: si_snr 
+    conf:
+      eps: 1.0e-7
+    wrapper: multilayer_pit
+    wrapper_conf:
+      weight: 1.0
+      independent_perm: True
diff --git a/espnet2/enh/layers/dpmulcat.py b/espnet2/enh/layers/dpmulcat.py
new file mode 100644
index 00000000000..f4a99c6b980
--- /dev/null
+++ b/espnet2/enh/layers/dpmulcat.py
@@ -0,0 +1,158 @@
+import torch
+import torch.nn as nn
+
+
+class MulCatBlock(nn.Module):
+    """The MulCat block that multiplies two RNN layers' outputs and concatenate it with the input Tensor.
+
+    Args:
+        input_size: int, dimension of the input feature. The input should have shape
+            (batch, seq_len, input_size).
+        hidden_size: int, dimension of the hidden state.
+        dropout: float, the dropout rate in the LSTM layer. (Default: 0.0)
+        bidirectional: bool, whether the RNN layers are bidirectional. (Default: True)
+    """
+    def __init__(self, input_size: int, hidden_size: int, dropout: float = 0.0, bidirectional: bool = True):
+        super(MulCatBlock, self).__init__()
+
+        num_direction = int(bidirectional) + 1
+
+        self.rnn = nn.LSTM(
+            input_size,
+            hidden_size,
+            1,
+            dropout=dropout,
+            batch_first=True,
+            bidirectional=bidirectional,
+        )
+        self.rnn_proj = nn.Linear(hidden_size * num_direction, input_size)
+
+        self.gate_rnn = nn.LSTM(
+            input_size,
+            hidden_size,
+            num_layers=1,
+            batch_first=True,
+            dropout=dropout,
+            bidirectional=bidirectional,
+        )
+        self.gate_rnn_proj = nn.Linear(hidden_size * num_direction, input_size)
+
+        self.block_projection = nn.Linear(input_size * 2, input_size)
+
+    def forward(self, input):
+        orig_shape = input.shape
+        # run rnn module
+        rnn_output, _ = self.rnn(input)
+        rnn_output = (
+            self.rnn_proj(rnn_output.contiguous().view(-1, rnn_output.shape[2]))
+            .view(orig_shape)
+            .contiguous()
+        )
+        # run gate rnn module
+        gate_rnn_output, _ = self.gate_rnn(input)
+        gate_rnn_output = (
+            self.gate_rnn_proj(
+                gate_rnn_output.contiguous().view(-1, gate_rnn_output.shape[2])
+            )
+            .view(orig_shape)
+            .contiguous()
+        )
+        # apply gated rnn
+        gated_output = torch.mul(rnn_output, gate_rnn_output)
+        # concatenate the input with rnn output
+        gated_output = torch.cat([gated_output, input], 2)
+        # linear projection to make the output shape the same as input
+        gated_output = self.block_projection(
+            gated_output.contiguous().view(-1, gated_output.shape[2])
+        ).view(orig_shape)
+        return gated_output
+
+
+class DPMulCat(nn.Module):
+    """Dual-path RNN module with MulCat blocks.
+
+    Args:
+        input_size: int, dimension of the input feature. The input should have shape
+            (batch, seq_len, input_size).
+        hidden_size: int, dimension of the hidden state.
+        output_size: int, dimension of the output size.
+        num_spk: int, the number of speakers in the output.
+        dropout: float, the dropout rate in the LSTM layer. (Default: 0.0)
+        bidirectional: bool, whether the RNN layers are bidirectional. (Default: True)
+        num_layers: int, number of stacked MulCat blocks. (Default: 4)
+        input_normalize: bool, whether to apply GroupNorm on the input Tensor. (Default: False)
+    """
+    def __init__(
+        self,
+        input_size: int,
+        hidden_size: int,
+        output_size: int,
+        num_spk: int,
+        dropout: float = 0.0,
+        num_layers: int = 4,
+        bidirectional: bool = True,
+        input_normalize: bool = False,
+    ):
+        super(DPMulCat, self).__init__()
+
+        self.rows_grnn = nn.ModuleList([])
+        self.cols_grnn = nn.ModuleList([])
+        self.rows_normalization = nn.ModuleList([])
+        self.cols_normalization = nn.ModuleList([])
+
+        # create the dual path pipeline
+        for i in range(num_layers):
+            self.rows_grnn.append(
+                MulCatBlock(
+                    input_size, hidden_size, dropout, bidirectional=bidirectional
+                )
+            )
+            self.cols_grnn.append(
+                MulCatBlock(
+                    input_size, hidden_size, dropout, bidirectional=bidirectional
+                )
+            )
+            if input_normalize:
+                self.rows_normalization.append(nn.GroupNorm(1, input_size, eps=1e-8))
+                self.cols_normalization.append(nn.GroupNorm(1, input_size, eps=1e-8))
+            else:
+                # used to disable normalization
+                self.rows_normalization.append(nn.Identity())
+                self.cols_normalization.append(nn.Identity())
+
+        self.output = nn.Sequential(
+            nn.PReLU(), nn.Conv2d(input_size, output_size * num_spk, 1)
+        )
+
+    def forward(self, input):
+        batch_size, _, d1, d2 = input.shape
+        output = input
+        output_all = []
+        for i in range(len(self.rows_grnn)):
+            row_input = (
+                output.permute(0, 3, 2, 1).contiguous().view(batch_size * d2, d1, -1)
+            )
+            row_output = self.rows_grnn[i](row_input)
+            row_output = (
+                row_output.view(batch_size, d2, d1, -1).permute(0, 3, 2, 1).contiguous()
+            )
+            row_output = self.rows_normalization[i](row_output)
+            # apply a skip connection
+            output += row_output
+            col_input = (
+                output.permute(0, 2, 3, 1).contiguous().view(batch_size * d1, d2, -1)
+            )
+            col_output = self.cols_grnn[i](col_input)
+            col_output = (
+                col_output.view(batch_size, d1, d2, -1).permute(0, 3, 1, 2).contiguous()
+            )
+            col_output = self.cols_normalization[i](col_output).contiguous()
+            # apply a skip connection
+            output += col_output
+
+            # if training mode, it returns the output Tensor from all layers.
+            # Otherwise, it only returns the one from the last layer.
+            if self.training or i == (len(self.rows_grnn) - 1):
+                output_i = self.output(output)
+                output_all.append(output_i)
+        return output_all
diff --git a/espnet2/enh/loss/wrappers/multilayer_pit_solver.py b/espnet2/enh/loss/wrappers/multilayer_pit_solver.py
new file mode 100644
index 00000000000..c4033d02ec9
--- /dev/null
+++ b/espnet2/enh/loss/wrappers/multilayer_pit_solver.py
@@ -0,0 +1,36 @@
+from itertools import permutations
+
+import torch
+
+from espnet2.enh.loss.criterions.abs_loss import AbsEnhLoss
+from espnet2.enh.loss.wrappers.abs_wrapper import AbsLossWrapper
+from espnet2.enh.loss.wrappers.pit_solver import PITSolver
+
+
+class MultiLayerPITSolver(AbsLossWrapper):
+    def __init__(self, criterion: AbsEnhLoss, weight=1.0, independent_perm=True,):
+        super().__init__()
+        self.criterion = criterion
+        self.weight = weight
+        self.independent_perm = independent_perm
+        self.solver = PITSolver(criterion, weight, independent_perm)
+
+    def forward(self, ref, infs, others={}):
+        """Permutation invariant training solver.
+
+        Args:
+            ref (List[torch.Tensor]): [(batch, ...), ...] x n_spk
+            inf (List[torch.Tensor]): [(batch, ...), ...]
+
+        Returns:
+            loss: (torch.Tensor): minimum loss with the best permutation
+            stats: dict, for collecting training status
+            others: dict, in this PIT solver, permutation order will be returned
+        """
+        losses = 0.0
+        for idx, inf in enumerate(infs):
+            loss, stats, others = self.solver(ref, inf, others)
+            losses = loss * (idx + 1) * (1.0 / len(infs))
+        losses = losses / len(infs)
+        return losses, stats, others
+        
diff --git a/espnet2/enh/separator/svoice_separator.py b/espnet2/enh/separator/svoice_separator.py
new file mode 100644
index 00000000000..0f385c5eb28
--- /dev/null
+++ b/espnet2/enh/separator/svoice_separator.py
@@ -0,0 +1,179 @@
+from collections import OrderedDict
+from typing import List
+from typing import Tuple
+from typing import Union
+import math
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from espnet2.enh.layers.dpmulcat import DPMulCat
+from espnet2.enh.layers.dprnn import split_feature, merge_feature
+from espnet2.enh.separator.abs_separator import AbsSeparator
+
+
+def overlap_and_add(signal, frame_step):
+    """Reconstructs a signal from a framed representation.
+    Adds potentially overlapping frames of a signal with shape
+    `[..., frames, frame_length]`, offsetting subsequent frames by `frame_step`.
+    The resulting tensor has shape `[..., output_size]` where
+        output_size = (frames - 1) * frame_step + frame_length
+    Args:
+        signal: A [..., frames, frame_length] Tensor. All dimensions may be unknown, and rank must be at least 2.
+        frame_step: An integer denoting overlap offsets. Must be less than or equal to frame_length.
+    Returns:
+        A Tensor with shape [..., output_size] containing the overlap-added frames of signal's inner-most two dimensions.
+        output_size = (frames - 1) * frame_step + frame_length
+    Based on https://github.com/tensorflow/tensorflow/blob/r1.12/tensorflow/contrib/signal/python/ops/reconstruction_ops.py
+    """
+    outer_dimensions = signal.size()[:-2]
+    frames, frame_length = signal.size()[-2:]
+
+    # gcd=Greatest Common Divisor
+    subframe_length = math.gcd(frame_length, frame_step)
+    subframe_step = frame_step // subframe_length
+    subframes_per_frame = frame_length // subframe_length
+    output_size = frame_step * (frames - 1) + frame_length
+    output_subframes = output_size // subframe_length
+
+    subframe_signal = signal.view(*outer_dimensions, -1, subframe_length)
+
+    frame = torch.arange(0, output_subframes).unfold(
+        0, subframes_per_frame, subframe_step
+    )
+    frame = frame.clone().detach().long().to(signal.device)
+    # frame = signal.new_tensor(frame).clone().long()  # signal may in GPU or CPU
+    frame = frame.contiguous().view(-1)
+
+    result = signal.new_zeros(*outer_dimensions, output_subframes, subframe_length)
+    result.index_add_(-2, frame, subframe_signal)
+    result = result.view(*outer_dimensions, -1)
+    return result
+
+
+class Encoder(nn.Module):
+    def __init__(self, enc_kernel_size: int, enc_feat_dim: int):
+        super().__init__()
+        # setting 50% overlap
+        self.conv = nn.Conv1d(
+            1,
+            enc_feat_dim,
+            kernel_size=enc_kernel_size,
+            stride=enc_kernel_size // 2,
+            bias=False,
+        )
+        self.nonlinear = nn.ReLU()
+
+    def forward(self, mixture):
+        mixture = torch.unsqueeze(mixture, 1)
+        mixture_w = self.nonlinear(self.conv(mixture))
+        return mixture_w
+
+
+class Decoder(nn.Module):
+    def __init__(self, kernel_size):
+        super().__init__()
+        self.kernel_size = kernel_size
+
+    def forward(self, est_source):
+        est_source = torch.transpose(est_source, 2, 3)
+        est_source = nn.AvgPool2d((1, self.kernel_size))(est_source)
+        est_source = overlap_and_add(est_source, self.kernel_size // 2)
+
+        return est_source
+
+
+class SVoiceSeparator(AbsSeparator):
+    """SVoice model for speech separation.
+
+    Args:
+        enc_dim: int, dimension of the encoder module's output. (Default: 128)
+        kernel_size: int, the kernel size of Conv1D layer in both encoder and
+            decoder modules. (Default: 8)
+        hidden_size: int, dimension of the hidden state in RNN layers. (Default: 128)
+        num_spk: int, the number of speakers in the output. (Default: 2)
+        num_layers: int, number of stacked MulCat blocks. (Default: 4)
+        segment_size: dual-path segment size. (Default: 20)
+        bidirectional: bool, whether the RNN layers are bidirectional. (Default: True)
+        input_normalize: bool, whether to apply GroupNorm on the input Tensor. (Default: False)
+    """
+
+    def __init__(
+        self,
+        placeholder: int,
+        enc_dim: int,
+        kernel_size: int,
+        hidden_size: int,
+        num_spk: int = 2,
+        num_layers: int = 4,
+        segment_size: int = 20,
+        bidirectional: bool = True,
+        input_normalize: bool = False,
+    ):
+        super().__init__()
+
+        self._num_spk = num_spk
+        self.enc_dim = enc_dim
+        self.segment_size = segment_size
+        # model sub-networks
+        self.encoder = Encoder(kernel_size, enc_dim)
+        self.decoder = Decoder(kernel_size)
+        self.rnn_model = DPMulCat(
+            input_size=enc_dim,
+            hidden_size=hidden_size,
+            output_size=enc_dim,
+            num_spk=num_spk,
+            num_layers=num_layers,
+            bidirectional=bidirectional,
+            input_normalize=input_normalize,
+        )
+
+    def forward(
+        self, input: torch.Tensor, ilens: torch.Tensor
+    ) -> Tuple[List[torch.Tensor], torch.Tensor, OrderedDict]:
+        """Forward.
+
+        Args:
+            input (torch.Tensor or ComplexTensor): Encoded feature [B, T, N]
+            ilens (torch.Tensor): input lengths [Batch]
+
+        Returns:
+            masked (List[Union(torch.Tensor, ComplexTensor)]): [(B, T, N), ...]
+            ilens (torch.Tensor): (B,)
+            others predicted data, e.g. masks: OrderedDict[
+                'mask_spk1': torch.Tensor(Batch, Frames, Freq),
+                'mask_spk2': torch.Tensor(Batch, Frames, Freq),
+                ...
+                'mask_spkn': torch.Tensor(Batch, Frames, Freq),
+            ]
+        """
+        # fix time dimension, might change due to convolution operations
+        T_mix = input.size(-1)
+
+        mixture_w = self.encoder(input)
+
+        enc_segments, enc_rest = split_feature(mixture_w, self.segment_size)
+        # separate
+        output_all = self.rnn_model(enc_segments)
+
+        # generate wav after each RNN block and optimize the loss
+        outputs = []
+        for ii in range(len(output_all)):
+            output_ii = merge_feature(output_all[ii], enc_rest)
+            output_ii = output_ii.view(
+                input.shape[0], self._num_spk, self.enc_dim, mixture_w.shape[2]
+            )
+            output_ii = self.decoder(output_ii)
+            T_est = output_ii.size(-1)
+            output_ii = F.pad(output_ii, (0, T_mix - T_est))
+            output_ii = [est for est in output_ii.unbind(dim=1)]
+            if self.training:
+                outputs.append(output_ii)
+            else:
+                outputs = output_ii
+        
+        others = {}
+        return outputs, ilens, others
+
+    @property
+    def num_spk(self):
+        return self._num_spk
\ No newline at end of file
diff --git a/espnet2/tasks/enh.py b/espnet2/tasks/enh.py
index c2ba18b4c01..8e086a46772 100644
--- a/espnet2/tasks/enh.py
+++ b/espnet2/tasks/enh.py
@@ -28,6 +28,7 @@
 from espnet2.enh.loss.criterions.time_domain import SNRLoss
 from espnet2.enh.loss.wrappers.abs_wrapper import AbsLossWrapper
 from espnet2.enh.loss.wrappers.fixed_order import FixedOrderSolver
+from espnet2.enh.loss.wrappers.multilayer_pit_solver import MultiLayerPITSolver
 from espnet2.enh.loss.wrappers.pit_solver import PITSolver
 from espnet2.enh.separator.abs_separator import AbsSeparator
 from espnet2.enh.separator.asteroid_models import AsteroidModel_Converter
@@ -39,6 +40,7 @@
 from espnet2.enh.separator.neural_beamformer import NeuralBeamformer
 from espnet2.enh.separator.rnn_separator import RNNSeparator
 from espnet2.enh.separator.skim_separator import SkiMSeparator
+from espnet2.enh.separator.svoice_separator import SVoiceSeparator
 from espnet2.enh.separator.tcn_separator import TCNSeparator
 from espnet2.enh.separator.transformer_separator import TransformerSeparator
 from espnet2.tasks.abs_task import AbsTask
@@ -63,6 +65,7 @@
     classes=dict(
         rnn=RNNSeparator,
         skim=SkiMSeparator,
+        svoice=SVoiceSeparator,
         tcn=TCNSeparator,
         dc_crn=DC_CRNSeparator,
         dprnn=DPRNNSeparator,
@@ -86,7 +89,7 @@
 
 loss_wrapper_choices = ClassChoices(
     name="loss_wrappers",
-    classes=dict(pit=PITSolver, fixed_order=FixedOrderSolver),
+    classes=dict(pit=PITSolver, fixed_order=FixedOrderSolver, multilayer_pit=MultiLayerPITSolver),
     type_check=AbsLossWrapper,
     default=None,
 )

From 59edb7c20b6283bfec1891d6c78ca2552a284c7f Mon Sep 17 00:00:00 2001
From: Zhaoheng Ni <zni@fb.com>
Date: Mon, 11 Apr 2022 13:55:52 +0100
Subject: [PATCH 21/56] address linter

---
 espnet2/enh/layers/dpmulcat.py            | 16 +++++++++++++---
 espnet2/enh/separator/svoice_separator.py | 21 +++++++++++++--------
 2 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/espnet2/enh/layers/dpmulcat.py b/espnet2/enh/layers/dpmulcat.py
index f4a99c6b980..9a0eb7b29d6 100644
--- a/espnet2/enh/layers/dpmulcat.py
+++ b/espnet2/enh/layers/dpmulcat.py
@@ -3,7 +3,8 @@
 
 
 class MulCatBlock(nn.Module):
-    """The MulCat block that multiplies two RNN layers' outputs and concatenate it with the input Tensor.
+    """The MulCat block that multiplies two RNN layers' outputs
+        and concatenate it with the input Tensor.
 
     Args:
         input_size: int, dimension of the input feature. The input should have shape
@@ -12,7 +13,14 @@ class MulCatBlock(nn.Module):
         dropout: float, the dropout rate in the LSTM layer. (Default: 0.0)
         bidirectional: bool, whether the RNN layers are bidirectional. (Default: True)
     """
-    def __init__(self, input_size: int, hidden_size: int, dropout: float = 0.0, bidirectional: bool = True):
+
+    def __init__(
+        self,
+        input_size: int,
+        hidden_size: int,
+        dropout: float = 0.0,
+        bidirectional: bool = True,
+    ):
         super(MulCatBlock, self).__init__()
 
         num_direction = int(bidirectional) + 1
@@ -80,8 +88,10 @@ class DPMulCat(nn.Module):
         dropout: float, the dropout rate in the LSTM layer. (Default: 0.0)
         bidirectional: bool, whether the RNN layers are bidirectional. (Default: True)
         num_layers: int, number of stacked MulCat blocks. (Default: 4)
-        input_normalize: bool, whether to apply GroupNorm on the input Tensor. (Default: False)
+        input_normalize: bool, whether to apply GroupNorm on the input Tensor.
+            (Default: False)
     """
+
     def __init__(
         self,
         input_size: int,
diff --git a/espnet2/enh/separator/svoice_separator.py b/espnet2/enh/separator/svoice_separator.py
index 0f385c5eb28..e0bdcf79502 100644
--- a/espnet2/enh/separator/svoice_separator.py
+++ b/espnet2/enh/separator/svoice_separator.py
@@ -1,11 +1,12 @@
+import math
 from collections import OrderedDict
 from typing import List
 from typing import Tuple
-from typing import Union
-import math
+
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
+
 from espnet2.enh.layers.dpmulcat import DPMulCat
 from espnet2.enh.layers.dprnn import split_feature, merge_feature
 from espnet2.enh.separator.abs_separator import AbsSeparator
@@ -18,10 +19,13 @@ def overlap_and_add(signal, frame_step):
     The resulting tensor has shape `[..., output_size]` where
         output_size = (frames - 1) * frame_step + frame_length
     Args:
-        signal: A [..., frames, frame_length] Tensor. All dimensions may be unknown, and rank must be at least 2.
-        frame_step: An integer denoting overlap offsets. Must be less than or equal to frame_length.
+        signal: A [..., frames, frame_length] Tensor. All dimensions may be unknown,
+            and rank must be at least 2.
+        frame_step: An integer denoting overlap offsets. Must be less than or equal
+            to frame_length.
     Returns:
-        A Tensor with shape [..., output_size] containing the overlap-added frames of signal's inner-most two dimensions.
+        A Tensor with shape [..., output_size] containing the overlap-added frames of
+        signal's inner-most two dimensions.
         output_size = (frames - 1) * frame_step + frame_length
     Based on https://github.com/tensorflow/tensorflow/blob/r1.12/tensorflow/contrib/signal/python/ops/reconstruction_ops.py
     """
@@ -94,7 +98,8 @@ class SVoiceSeparator(AbsSeparator):
         num_layers: int, number of stacked MulCat blocks. (Default: 4)
         segment_size: dual-path segment size. (Default: 20)
         bidirectional: bool, whether the RNN layers are bidirectional. (Default: True)
-        input_normalize: bool, whether to apply GroupNorm on the input Tensor. (Default: False)
+        input_normalize: bool, whether to apply GroupNorm on the input Tensor.
+            (Default: False)
     """
 
     def __init__(
@@ -170,10 +175,10 @@ def forward(
                 outputs.append(output_ii)
             else:
                 outputs = output_ii
-        
+
         others = {}
         return outputs, ilens, others
 
     @property
     def num_spk(self):
-        return self._num_spk
\ No newline at end of file
+        return self._num_spk

From 5ae7c9580f85dae5bc81cb1e845366c251d871ac Mon Sep 17 00:00:00 2001
From: Zhaoheng Ni <zni@fb.com>
Date: Tue, 12 Apr 2022 17:48:13 +0100
Subject: [PATCH 22/56] fix docstring

---
 espnet2/enh/loss/wrappers/multilayer_pit_solver.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/espnet2/enh/loss/wrappers/multilayer_pit_solver.py b/espnet2/enh/loss/wrappers/multilayer_pit_solver.py
index c4033d02ec9..72679159196 100644
--- a/espnet2/enh/loss/wrappers/multilayer_pit_solver.py
+++ b/espnet2/enh/loss/wrappers/multilayer_pit_solver.py
@@ -1,14 +1,15 @@
-from itertools import permutations
-
-import torch
-
 from espnet2.enh.loss.criterions.abs_loss import AbsEnhLoss
 from espnet2.enh.loss.wrappers.abs_wrapper import AbsLossWrapper
 from espnet2.enh.loss.wrappers.pit_solver import PITSolver
 
 
 class MultiLayerPITSolver(AbsLossWrapper):
-    def __init__(self, criterion: AbsEnhLoss, weight=1.0, independent_perm=True,):
+    def __init__(
+        self,
+        criterion: AbsEnhLoss,
+        weight=1.0,
+        independent_perm=True,
+    ):
         super().__init__()
         self.criterion = criterion
         self.weight = weight
@@ -20,7 +21,7 @@ def forward(self, ref, infs, others={}):
 
         Args:
             ref (List[torch.Tensor]): [(batch, ...), ...] x n_spk
-            inf (List[torch.Tensor]): [(batch, ...), ...]
+            infs (List[List[torch.Tensor]]): [(batch, ...), ...] x n_layer
 
         Returns:
             loss: (torch.Tensor): minimum loss with the best permutation
@@ -33,4 +34,3 @@ def forward(self, ref, infs, others={}):
             losses = loss * (idx + 1) * (1.0 / len(infs))
         losses = losses / len(infs)
         return losses, stats, others
-        

From 87f73951acf908f91833964bea71729f061f8185 Mon Sep 17 00:00:00 2001
From: Zhaoheng Ni <zni@fb.com>
Date: Thu, 14 Apr 2022 15:18:36 +0100
Subject: [PATCH 23/56] add results, fix inference mode

---
 egs2/wsj0_2mix/enh1/README.md                 | 22 +++++++++++++++++++
 .../loss/wrappers/multilayer_pit_solver.py    | 19 +++++++++++-----
 espnet2/enh/separator/svoice_separator.py     | 17 +++++++-------
 3 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/egs2/wsj0_2mix/enh1/README.md b/egs2/wsj0_2mix/enh1/README.md
index b5040ed04ae..c61caed5d56 100644
--- a/egs2/wsj0_2mix/enh1/README.md
+++ b/egs2/wsj0_2mix/enh1/README.md
@@ -91,3 +91,25 @@ config: conf/tuning/train_enh_dc_crn_mapping_snr.yaml
 |---|---|---|---|---|---|
 |enhanced_cv_min_8k|0.93|14.86|14.16|24.02|13.73|
 |enhanced_tt_min_8k|0.94|14.25|13.46|23.13|13.01|
+
+<!-- Generated by ./scripts/utils/show_enh_score.sh -->
+# RESULTS
+## Environments
+- date: `Thu Apr 14 09:47:05 UTC 2022`
+- python version: `3.8.12 (default, Oct 12 2021, 13:49:34)  [GCC 7.5.0]`
+- espnet version: `espnet 0.10.7a1`
+- pytorch version: `pytorch 1.10.1+cu111`
+- Git hash: `9dbe4179b866b994f6914ef52ea7483696d22760`
+  - Commit date: `Wed Mar 16 13:25:26 2022 +0000`
+
+
+## svoice (multi-layer Si-SNR loss)
+
+ - config: conf/tuning/train_enh_svoice.yaml
+ - Pretrained model: https://huggingface.co/Zhaoheng/svoice_wsj0_2mix
+
+|dataset|STOI|SAR|SDR|SIR|
+|---|---|---|---|---|
+|enhanced_cv_min_8k|0.97|21.44|20.98|32.21|
+|enhanced_tt_min_8k|0.98|21.41|20.96|32.27|
+
diff --git a/espnet2/enh/loss/wrappers/multilayer_pit_solver.py b/espnet2/enh/loss/wrappers/multilayer_pit_solver.py
index 72679159196..bbb2f987207 100644
--- a/espnet2/enh/loss/wrappers/multilayer_pit_solver.py
+++ b/espnet2/enh/loss/wrappers/multilayer_pit_solver.py
@@ -1,3 +1,5 @@
+import torch
+
 from espnet2.enh.loss.criterions.abs_loss import AbsEnhLoss
 from espnet2.enh.loss.wrappers.abs_wrapper import AbsLossWrapper
 from espnet2.enh.loss.wrappers.pit_solver import PITSolver
@@ -21,7 +23,8 @@ def forward(self, ref, infs, others={}):
 
         Args:
             ref (List[torch.Tensor]): [(batch, ...), ...] x n_spk
-            infs (List[List[torch.Tensor]]): [(batch, ...), ...] x n_layer
+            infs (Union[List[torch.Tensor], List[List[torch.Tensor]]]):
+                [(batch, ...), ...]
 
         Returns:
             loss: (torch.Tensor): minimum loss with the best permutation
@@ -29,8 +32,14 @@ def forward(self, ref, infs, others={}):
             others: dict, in this PIT solver, permutation order will be returned
         """
         losses = 0.0
-        for idx, inf in enumerate(infs):
-            loss, stats, others = self.solver(ref, inf, others)
-            losses = loss * (idx + 1) * (1.0 / len(infs))
-        losses = losses / len(infs)
+        # In validation, the model only estimates waveforms in the last layer.
+        # The shape of infs is List[torch.Tensor]
+        if torch.is_tensor(infs[0]) and len(infs) == len(ref):
+            loss, stats, others = self.solver(ref, infs, others)
+            losses = loss
+        else:
+            for idx, inf in enumerate(infs):
+                loss, stats, others = self.solver(ref, inf, others)
+                losses = losses + loss * (idx + 1) * (1.0 / len(infs))
+            losses = losses / len(infs)
         return losses, stats, others
diff --git a/espnet2/enh/separator/svoice_separator.py b/espnet2/enh/separator/svoice_separator.py
index e0bdcf79502..d4aaf0fdce4 100644
--- a/espnet2/enh/separator/svoice_separator.py
+++ b/espnet2/enh/separator/svoice_separator.py
@@ -1,12 +1,10 @@
-import math
 from collections import OrderedDict
 from typing import List
 from typing import Tuple
-
+import math
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
-
 from espnet2.enh.layers.dpmulcat import DPMulCat
 from espnet2.enh.layers.dprnn import split_feature, merge_feature
 from espnet2.enh.separator.abs_separator import AbsSeparator
@@ -21,13 +19,14 @@ def overlap_and_add(signal, frame_step):
     Args:
         signal: A [..., frames, frame_length] Tensor. All dimensions may be unknown,
             and rank must be at least 2.
-        frame_step: An integer denoting overlap offsets. Must be less than or equal
-            to frame_length.
+        frame_step: An integer denoting overlap offsets.
+            Must be less than or equal to frame_length.
     Returns:
-        A Tensor with shape [..., output_size] containing the overlap-added frames of
-        signal's inner-most two dimensions.
+        A Tensor with shape [..., output_size] containing the
+            overlap-added frames of signal's inner-most two dimensions.
         output_size = (frames - 1) * frame_step + frame_length
-    Based on https://github.com/tensorflow/tensorflow/blob/r1.12/tensorflow/contrib/signal/python/ops/reconstruction_ops.py
+    Based on
+        https://github.com/tensorflow/tensorflow/blob/r1.12/tensorflow/contrib/signal/python/ops/reconstruction_ops.py
     """
     outer_dimensions = signal.size()[:-2]
     frames, frame_length = signal.size()[-2:]
@@ -170,7 +169,7 @@ def forward(
             output_ii = self.decoder(output_ii)
             T_est = output_ii.size(-1)
             output_ii = F.pad(output_ii, (0, T_mix - T_est))
-            output_ii = [est for est in output_ii.unbind(dim=1)]
+            output_ii = list(output_ii.unbind(dim=1))
             if self.training:
                 outputs.append(output_ii)
             else:

From a071e9a8c87d2ba03b1828c6c136fcbfe9667ef2 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Thu, 14 Apr 2022 22:54:30 +0800
Subject: [PATCH 24/56] Rewrite IBM and IRM logic

---
 espnet2/enh/loss/criterions/tf_domain.py | 13 +++---
 test/espnet2/enh/test_espnet_model.py    | 51 +-----------------------
 2 files changed, 9 insertions(+), 55 deletions(-)

diff --git a/espnet2/enh/loss/criterions/tf_domain.py b/espnet2/enh/loss/criterions/tf_domain.py
index 29a44c729b7..0b3012bb49a 100644
--- a/espnet2/enh/loss/criterions/tf_domain.py
+++ b/espnet2/enh/loss/criterions/tf_domain.py
@@ -43,20 +43,21 @@ def _create_mask_label(mix_spec, ref_spec, noise_spec=None, mask_type="IAM"):
     mask_label = []
     if ref_spec[0].ndim < mix_spec.ndim:
         # (B, T, F) -> (B, T, 1, F)
-        ref_spec = [r.unsqueeze(2) for r in ref_spec]
+        ref_spec = [r.unsqueeze(2).expand_as(mix_spec.real) for r in ref_spec]
     for idx, r in enumerate(ref_spec):
         mask = None
         if mask_type == "IBM":
             if noise_spec is None:
-                raise ValueError("noise_spec should not be None")
-            flags = [abs(r) >= abs(n) for n in ref_spec + [noise_spec]]
+                flags = [abs(r) >= abs(n) for n in ref_spec]
+            else:
+                flags = [abs(r) >= abs(n) for n in ref_spec + [noise_spec]]
             mask = reduce(lambda x, y: x * y, flags)
             mask = mask.int()
         elif mask_type == "IRM":
-            if noise_spec is None:
-                raise ValueError("noise_spec should not be None")
             beta = 0.5
-            res_spec = sum(n for i, n in enumerate(ref_spec) if i != idx) + noise_spec
+            res_spec = sum(n for i, n in enumerate(ref_spec) if i != idx)
+            if noise_spec is not None:
+                res_spec += noise_spec
             mask = (abs(r).pow(2) / (abs(res_spec).pow(2) + EPS)).pow(beta)
         elif mask_type == "IAM":
             mask = abs(r) / (abs(mix_spec) + EPS)
diff --git a/test/espnet2/enh/test_espnet_model.py b/test/espnet2/enh/test_espnet_model.py
index c329105ecb4..53429526ef6 100644
--- a/test/espnet2/enh/test_espnet_model.py
+++ b/test/espnet2/enh/test_espnet_model.py
@@ -253,55 +253,8 @@ def test_forward_with_beamformer_net(
         "speech_mix": inputs,
         "speech_mix_lengths": ilens,
         **{"speech_ref{}".format(i + 1): speech_refs[i] for i in range(num_spk)},
-        "noise_ref1": noise_ref1,
         "dereverb_ref1": dereverb_ref1,
     }
     loss, stats, weight = enh_model(**kwargs)
-
-
-@pytest.mark.parametrize("mask_type", ["IBM", "IRM"])
-@pytest.mark.parametrize("num_spk", [1, 2, 3])
-def test_invalid_case_for_masks(mask_type, num_spk):
-    ch = 3
-    inputs = random_speech[..., :ch].float()
-    ilens = torch.LongTensor([16, 12])
-    speech_refs = [torch.randn(2, 16, ch, dtype=torch.float) for spk in range(num_spk)]
-
-    encoder = STFTEncoder(n_fft=8, hop_length=2, use_builtin_complex=False)
-    decoder = STFTDecoder(n_fft=8, hop_length=2)
-
-    beamformer = NeuralBeamformer(
-        input_dim=5,
-        loss_type="mask_mse",
-        num_spk=num_spk,
-        use_wpe=True,
-        wlayers=2,
-        wunits=2,
-        wprojs=2,
-        use_dnn_mask_for_wpe=True,
-        multi_source_wpe=True,
-        use_beamformer=True,
-        blayers=2,
-        bunits=2,
-        bprojs=2,
-        badim=2,
-        ref_channel=0,
-        use_noise_mask=False,
-        beamformer_type="mvdr_souden",
-    )
-    enh_model = ESPnetEnhancementModel(
-        encoder=encoder,
-        decoder=decoder,
-        separator=beamformer,
-        loss_type="mask_mse",
-        mask_type=mask_type,
-        loss_wrappers=[pit_wrapper],
-    )
-
-    kwargs = {
-        "speech_mix": inputs,
-        "speech_mix_lengths": ilens,
-        **{"speech_ref{}".format(i + 1): speech_refs[i] for i in range(num_spk)},
-    }
-    with pytest.raises(ValueError):
-        loss, stats, weight = enh_model(**kwargs)
+    if mask_type in ("IBM", "IRM"):
+        loss, stats, weight = enh_model(**kwargs, noise_ref1=noise_ref1)

From d91ad1638c8df69b71566867fcd8188fb57a6bff Mon Sep 17 00:00:00 2001
From: Zhaoheng Ni <zni@fb.com>
Date: Fri, 15 Apr 2022 17:50:39 +0100
Subject: [PATCH 25/56] address comments, add unit tests

---
 egs2/wsj0_2mix/enh1/README.md                 |  8 +-
 espnet2/enh/layers/dpmulcat.py                | 41 ++++++--
 .../loss/wrappers/multilayer_pit_solver.py    | 20 +++-
 espnet2/enh/separator/svoice_separator.py     | 21 +++-
 .../wrappers/test_multilayer_pit_solver.py    | 52 ++++++++++
 .../enh/separator/test_svoice_separator.py    | 95 +++++++++++++++++++
 test/espnet2/enh/test_espnet_model.py         | 59 ++++++++++++
 7 files changed, 277 insertions(+), 19 deletions(-)
 create mode 100644 test/espnet2/enh/loss/wrappers/test_multilayer_pit_solver.py
 create mode 100644 test/espnet2/enh/separator/test_svoice_separator.py

diff --git a/egs2/wsj0_2mix/enh1/README.md b/egs2/wsj0_2mix/enh1/README.md
index c61caed5d56..38feb6ed3a7 100644
--- a/egs2/wsj0_2mix/enh1/README.md
+++ b/egs2/wsj0_2mix/enh1/README.md
@@ -108,8 +108,8 @@ config: conf/tuning/train_enh_dc_crn_mapping_snr.yaml
  - config: conf/tuning/train_enh_svoice.yaml
  - Pretrained model: https://huggingface.co/Zhaoheng/svoice_wsj0_2mix
 
-|dataset|STOI|SAR|SDR|SIR|
-|---|---|---|---|---|
-|enhanced_cv_min_8k|0.97|21.44|20.98|32.21|
-|enhanced_tt_min_8k|0.98|21.41|20.96|32.27|
+|dataset|STOI|SAR|SDR|SIR|SI_SNR|
+|---|---|---|---|---|---|
+|enhanced_cv_min_8k|0.97|21.44|20.98|32.21|20.67|
+|enhanced_tt_min_8k|0.98|21.41|20.96|32.27|20.66|
 
diff --git a/espnet2/enh/layers/dpmulcat.py b/espnet2/enh/layers/dpmulcat.py
index 9a0eb7b29d6..60d6a54ceb1 100644
--- a/espnet2/enh/layers/dpmulcat.py
+++ b/espnet2/enh/layers/dpmulcat.py
@@ -3,12 +3,11 @@
 
 
 class MulCatBlock(nn.Module):
-    """The MulCat block that multiplies two RNN layers' outputs
-        and concatenate it with the input Tensor.
+    """The MulCat block.
 
     Args:
-        input_size: int, dimension of the input feature. The input should have shape
-            (batch, seq_len, input_size).
+        input_size: int, dimension of the input feature.
+            The input should have shape (batch, seq_len, input_size).
         hidden_size: int, dimension of the hidden state.
         dropout: float, the dropout rate in the LSTM layer. (Default: 0.0)
         bidirectional: bool, whether the RNN layers are bidirectional. (Default: True)
@@ -21,7 +20,7 @@ def __init__(
         dropout: float = 0.0,
         bidirectional: bool = True,
     ):
-        super(MulCatBlock, self).__init__()
+        super().__init__()
 
         num_direction = int(bidirectional) + 1
 
@@ -48,6 +47,16 @@ def __init__(
         self.block_projection = nn.Linear(input_size * 2, input_size)
 
     def forward(self, input):
+        """Compute output after MulCatBlock.
+
+        Args:
+            input (torch.Tensor): The input feature.
+                Tensor of shape (batch, time, feature_dim)
+
+        Returns:
+            (torch.Tensor): The output feature after MulCatBlock.
+                Tensor of shape (batch, time, feature_dim)
+        """
         orig_shape = input.shape
         # run rnn module
         rnn_output, _ = self.rnn(input)
@@ -80,8 +89,8 @@ class DPMulCat(nn.Module):
     """Dual-path RNN module with MulCat blocks.
 
     Args:
-        input_size: int, dimension of the input feature. The input should have shape
-            (batch, seq_len, input_size).
+        input_size: int, dimension of the input feature.
+            The input should have shape (batch, seq_len, input_size).
         hidden_size: int, dimension of the hidden state.
         output_size: int, dimension of the output size.
         num_spk: int, the number of speakers in the output.
@@ -103,7 +112,7 @@ def __init__(
         bidirectional: bool = True,
         input_normalize: bool = False,
     ):
-        super(DPMulCat, self).__init__()
+        super().__init__()
 
         self.rows_grnn = nn.ModuleList([])
         self.cols_grnn = nn.ModuleList([])
@@ -135,6 +144,18 @@ def __init__(
         )
 
     def forward(self, input):
+        """Compute output after DPMulCat module.
+
+        Args:
+            input (torch.Tensor): The input feature.
+                Tensor of shape (batch, N, dim1, dim2)
+                Apply RNN on dim1 first and then dim2
+
+        Returns:
+            (list(torch.Tensor) or list(list(torch.Tensor))
+                In training mode, the module returns output of each DPMulCat block.
+                In eval mode, the module only returns output in the last block.
+        """
         batch_size, _, d1, d2 = input.shape
         output = input
         output_all = []
@@ -148,7 +169,7 @@ def forward(self, input):
             )
             row_output = self.rows_normalization[i](row_output)
             # apply a skip connection
-            output += row_output
+            output = output + row_output
             col_input = (
                 output.permute(0, 2, 3, 1).contiguous().view(batch_size * d1, d2, -1)
             )
@@ -158,7 +179,7 @@ def forward(self, input):
             )
             col_output = self.cols_normalization[i](col_output).contiguous()
             # apply a skip connection
-            output += col_output
+            output = output + col_output
 
             # if training mode, it returns the output Tensor from all layers.
             # Otherwise, it only returns the one from the last layer.
diff --git a/espnet2/enh/loss/wrappers/multilayer_pit_solver.py b/espnet2/enh/loss/wrappers/multilayer_pit_solver.py
index bbb2f987207..34ad1d1f28b 100644
--- a/espnet2/enh/loss/wrappers/multilayer_pit_solver.py
+++ b/espnet2/enh/loss/wrappers/multilayer_pit_solver.py
@@ -12,6 +12,22 @@ def __init__(
         weight=1.0,
         independent_perm=True,
     ):
+        """Multi-Layer Permutation Invariant Training Solver.
+
+        Compute the PIT loss given inferences of multiple layers and a single reference.
+        It also support single inference and single reference in evaluation stage.
+
+        Args:
+            criterion (AbsEnhLoss): an instance of AbsEnhLoss
+            weight (float): weight (between 0 and 1) of current loss
+                for multi-task learning.
+            independent_perm (bool):
+                If True, PIT will be performed in forward to find the best permutation;
+                If False, the permutation from the last LossWrapper output will be
+                inherited.
+                Note: You should be careful about the ordering of loss
+                wrappers defined in the yaml config, if this argument is False.
+        """
         super().__init__()
         self.criterion = criterion
         self.weight = weight
@@ -32,11 +48,13 @@ def forward(self, ref, infs, others={}):
             others: dict, in this PIT solver, permutation order will be returned
         """
         losses = 0.0
-        # In validation, the model only estimates waveforms in the last layer.
+        # In single-layer case, the model only estimates waveforms in the last layer.
         # The shape of infs is List[torch.Tensor]
         if torch.is_tensor(infs[0]) and len(infs) == len(ref):
             loss, stats, others = self.solver(ref, infs, others)
             losses = loss
+        # In multi-layer case, weighted-sum the PIT loss of each layer
+        # The shape of ins is List[List[torch.Tensor]]
         else:
             for idx, inf in enumerate(infs):
                 loss, stats, others = self.solver(ref, inf, others)
diff --git a/espnet2/enh/separator/svoice_separator.py b/espnet2/enh/separator/svoice_separator.py
index d4aaf0fdce4..5da45851a85 100644
--- a/espnet2/enh/separator/svoice_separator.py
+++ b/espnet2/enh/separator/svoice_separator.py
@@ -1,32 +1,40 @@
 from collections import OrderedDict
+import math
 from typing import List
 from typing import Tuple
-import math
+
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
+
 from espnet2.enh.layers.dpmulcat import DPMulCat
-from espnet2.enh.layers.dprnn import split_feature, merge_feature
+from espnet2.enh.layers.dprnn import merge_feature
+from espnet2.enh.layers.dprnn import split_feature
 from espnet2.enh.separator.abs_separator import AbsSeparator
 
 
 def overlap_and_add(signal, frame_step):
     """Reconstructs a signal from a framed representation.
+
     Adds potentially overlapping frames of a signal with shape
     `[..., frames, frame_length]`, offsetting subsequent frames by `frame_step`.
     The resulting tensor has shape `[..., output_size]` where
         output_size = (frames - 1) * frame_step + frame_length
+
     Args:
         signal: A [..., frames, frame_length] Tensor. All dimensions may be unknown,
             and rank must be at least 2.
         frame_step: An integer denoting overlap offsets.
             Must be less than or equal to frame_length.
+
     Returns:
         A Tensor with shape [..., output_size] containing the
             overlap-added frames of signal's inner-most two dimensions.
         output_size = (frames - 1) * frame_step + frame_length
+
     Based on
-        https://github.com/tensorflow/tensorflow/blob/r1.12/tensorflow/contrib/signal/python/ops/reconstruction_ops.py
+
+https://github.com/tensorflow/tensorflow/blob/r1.12/tensorflow/contrib/signal/python/ops/reconstruction_ops.py
     """
     outer_dimensions = signal.size()[:-2]
     frames, frame_length = signal.size()[-2:]
@@ -88,6 +96,11 @@ def forward(self, est_source):
 class SVoiceSeparator(AbsSeparator):
     """SVoice model for speech separation.
 
+    Reference:
+        Voice Separation with an Unknown Number of Multiple Speakers;
+        E. Nachmani et al., 2020;
+        https://arxiv.org/abs/2003.01531
+
     Args:
         enc_dim: int, dimension of the encoder module's output. (Default: 128)
         kernel_size: int, the kernel size of Conv1D layer in both encoder and
@@ -103,7 +116,7 @@ class SVoiceSeparator(AbsSeparator):
 
     def __init__(
         self,
-        placeholder: int,
+        input_dim: int,
         enc_dim: int,
         kernel_size: int,
         hidden_size: int,
diff --git a/test/espnet2/enh/loss/wrappers/test_multilayer_pit_solver.py b/test/espnet2/enh/loss/wrappers/test_multilayer_pit_solver.py
new file mode 100644
index 00000000000..3505a007eee
--- /dev/null
+++ b/test/espnet2/enh/loss/wrappers/test_multilayer_pit_solver.py
@@ -0,0 +1,52 @@
+import pytest
+
+import torch
+
+from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainL1
+from espnet2.enh.loss.wrappers.multilayer_pit_solver import MultiLayerPITSolver
+
+
+@pytest.mark.parametrize("num_spk", [1, 2, 3])
+def test_MultiLayerPITSolver_forward_multi_layer(num_spk):
+
+    batch = 2
+    num_layers = 2
+    # infs is a List of List (num_layer x num_speaker Tensors)
+    infs = [
+        [torch.rand(batch, 10, 100) for spk in range(num_spk)]
+        for _ in range(num_layers)
+    ]
+    ref = [infs[-1][num_spk - spk - 1] for spk in range(num_spk)]  # reverse inf as ref
+    solver = MultiLayerPITSolver(FrequencyDomainL1(), independent_perm=True)
+
+    loss, stats, others = solver(ref, infs)
+    perm = others["perm"]
+    correct_perm = list(range(num_spk))
+    correct_perm.reverse()
+    assert perm[0].equal(torch.tensor(correct_perm))
+
+    # test for independent_perm is False
+
+    solver = MultiLayerPITSolver(FrequencyDomainL1(), independent_perm=False)
+    loss, stats, others = solver(ref, infs, {"perm": perm})
+
+
+@pytest.mark.parametrize("num_spk", [1, 2, 3])
+def test_MultiLayerPITSolver_forward_single_layer(num_spk):
+
+    batch = 2
+    # inf is a List of Tensors
+    inf = [torch.rand(batch, 10, 100) for spk in range(num_spk)]
+    ref = [inf[num_spk - spk - 1] for spk in range(num_spk)]  # reverse inf as ref
+    solver = MultiLayerPITSolver(FrequencyDomainL1(), independent_perm=True)
+
+    loss, stats, others = solver(ref, inf)
+    perm = others["perm"]
+    correct_perm = list(range(num_spk))
+    correct_perm.reverse()
+    assert perm[0].equal(torch.tensor(correct_perm))
+
+    # test for independent_perm is False
+
+    solver = MultiLayerPITSolver(FrequencyDomainL1(), independent_perm=False)
+    loss, stats, others = solver(ref, inf, {"perm": perm})
diff --git a/test/espnet2/enh/separator/test_svoice_separator.py b/test/espnet2/enh/separator/test_svoice_separator.py
new file mode 100644
index 00000000000..b5a7d39944d
--- /dev/null
+++ b/test/espnet2/enh/separator/test_svoice_separator.py
@@ -0,0 +1,95 @@
+import pytest
+
+import torch
+from torch import Tensor
+
+from espnet2.enh.separator.svoice_separator import SVoiceSeparator
+
+
+@pytest.mark.parametrize("input_dim", [1])
+@pytest.mark.parametrize("enc_dim", [4])
+@pytest.mark.parametrize("kernel_size", [4])
+@pytest.mark.parametrize("hidden_size", [4])
+@pytest.mark.parametrize("num_spk", [1, 2])
+@pytest.mark.parametrize("num_layers", [1, 2])
+@pytest.mark.parametrize("segment_size", [2])
+@pytest.mark.parametrize("bidirectional", [False])
+@pytest.mark.parametrize("input_normalize", [False])
+def test_fasnet_separator_forward_backward_real(
+    input_dim,
+    enc_dim,
+    kernel_size,
+    hidden_size,
+    num_spk,
+    num_layers,
+    segment_size,
+    bidirectional,
+    input_normalize,
+):
+    model = SVoiceSeparator(
+        input_dim=input_dim,
+        enc_dim=enc_dim,
+        kernel_size=kernel_size,
+        hidden_size=hidden_size,
+        num_spk=num_spk,
+        num_layers=num_layers,
+        segment_size=segment_size,
+        bidirectional=bidirectional,
+        input_normalize=input_normalize,
+    )
+    model.train()
+
+    x = torch.rand(2, 8000)
+    x_lens = torch.tensor([400, 300], dtype=torch.long)
+
+    separated, _, _ = model(x, ilens=x_lens)
+
+    assert isinstance(separated[0][0], Tensor)
+    assert len(separated) == num_layers
+
+    separated[0][0].mean().backward()
+
+
+def test_fasnet_separator_output_train():
+    x = torch.rand(2, 8000)
+    x_lens = torch.tensor([10, 8], dtype=torch.long)
+
+    for num_spk in range(1, 3):
+        model = SVoiceSeparator(
+            input_dim=12,
+            enc_dim=8,
+            kernel_size=8,
+            hidden_size=8,
+            num_spk=num_spk,
+            num_layers=4,
+            segment_size=2,
+            bidirectional=False,
+            input_normalize=False,
+        )
+        model.train()
+        waveforms, _, _ = model(x, x_lens)
+        assert isinstance(waveforms, list)
+        assert isinstance(waveforms[0], list)
+        assert x[0].shape == waveforms[0][0][0].shape
+
+
+def test_fasnet_separator_output_eval():
+    x = torch.rand(2, 8000)
+    x_lens = torch.tensor([10, 8], dtype=torch.long)
+
+    for num_spk in range(1, 3):
+        model = SVoiceSeparator(
+            input_dim=12,
+            enc_dim=8,
+            kernel_size=8,
+            hidden_size=8,
+            num_spk=num_spk,
+            num_layers=4,
+            segment_size=2,
+            bidirectional=False,
+            input_normalize=False,
+        )
+        model.eval()
+        waveforms, _, _ = model(x, x_lens)
+        assert isinstance(waveforms, list)
+        assert x[0].shape == waveforms[0][0].shape
diff --git a/test/espnet2/enh/test_espnet_model.py b/test/espnet2/enh/test_espnet_model.py
index f3a29f1757d..d1d1b3c0d4b 100644
--- a/test/espnet2/enh/test_espnet_model.py
+++ b/test/espnet2/enh/test_espnet_model.py
@@ -4,20 +4,24 @@
 import torch
 
 from espnet2.enh.decoder.conv_decoder import ConvDecoder
+from espnet2.enh.decoder.null_decoder import NullDecoder
 from espnet2.enh.decoder.stft_decoder import STFTDecoder
 from espnet2.enh.encoder.conv_encoder import ConvEncoder
+from espnet2.enh.encoder.null_encoder import NullEncoder
 from espnet2.enh.encoder.stft_encoder import STFTEncoder
 from espnet2.enh.espnet_model import ESPnetEnhancementModel
 from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainL1
 from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainMSE
 from espnet2.enh.loss.criterions.time_domain import SISNRLoss
 from espnet2.enh.loss.wrappers.fixed_order import FixedOrderSolver
+from espnet2.enh.loss.wrappers.multilayer_pit_solver import MultiLayerPITSolver
 from espnet2.enh.loss.wrappers.pit_solver import PITSolver
 from espnet2.enh.separator.dc_crn_separator import DC_CRNSeparator
 from espnet2.enh.separator.dccrn_separator import DCCRNSeparator
 from espnet2.enh.separator.dprnn_separator import DPRNNSeparator
 from espnet2.enh.separator.neural_beamformer import NeuralBeamformer
 from espnet2.enh.separator.rnn_separator import RNNSeparator
+from espnet2.enh.separator.svoice_separator import SVoiceSeparator
 from espnet2.enh.separator.tcn_separator import TCNSeparator
 from espnet2.enh.separator.transformer_separator import TransformerSeparator
 
@@ -53,6 +57,10 @@
     stride=18,
 )
 
+null_encoder = NullEncoder()
+
+null_decoder = NullDecoder()
+
 rnn_separator = RNNSeparator(
     input_dim=17,
     layer=1,
@@ -65,6 +73,18 @@
 
 dprnn_separator = DPRNNSeparator(input_dim=17, layer=1, unit=10, segment_size=4)
 
+svoice_separator = SVoiceSeparator(
+    input_dim=17,
+    enc_dim=4,
+    kernel_size=4,
+    hidden_size=4,
+    num_spk=2,
+    num_layers=2,
+    segment_size=4,
+    bidirectional=False,
+    input_normalize=False,
+)
+
 tcn_separator = TCNSeparator(
     input_dim=17,
     layer=2,
@@ -87,6 +107,7 @@
 tf_l1_loss = FrequencyDomainL1()
 
 pit_wrapper = PITSolver(criterion=si_snr_loss)
+multilayer_pit_solver = MultiLayerPITSolver(criterion=si_snr_loss)
 fix_order_solver = FixedOrderSolver(criterion=tf_mse_loss)
 
 
@@ -141,6 +162,44 @@ def test_single_channel_model(encoder, decoder, separator, training, loss_wrappe
     loss, stats, weight = enh_model(**kwargs)
 
 
+@pytest.mark.parametrize(
+    "encoder, decoder",
+    [
+        (null_encoder, null_decoder),
+    ],
+)
+@pytest.mark.parametrize(
+    "separator",
+    [
+        svoice_separator,
+    ],
+)
+@pytest.mark.parametrize("training", [True, False])
+@pytest.mark.parametrize("loss_wrappers", [[multilayer_pit_solver]])
+def test_svoice_model(encoder, decoder, separator, training, loss_wrappers):
+    inputs = torch.randn(2, 300)
+    ilens = torch.LongTensor([300, 200])
+    speech_refs = [torch.randn(2, 300).float(), torch.randn(2, 300).float()]
+    enh_model = ESPnetEnhancementModel(
+        encoder=encoder,
+        separator=separator,
+        decoder=decoder,
+        loss_wrappers=loss_wrappers,
+    )
+
+    if training:
+        enh_model.train()
+    else:
+        enh_model.eval()
+
+    kwargs = {
+        "speech_mix": inputs,
+        "speech_mix_lengths": ilens,
+        **{"speech_ref{}".format(i + 1): speech_refs[i] for i in range(2)},
+    }
+    loss, stats, weight = enh_model(**kwargs)
+
+
 random_speech = torch.tensor(
     [
         [

From 2b4161c98698786c26e12650f6d2a748a0532719 Mon Sep 17 00:00:00 2001
From: Zhaoheng Ni <zni@fb.com>
Date: Sat, 16 Apr 2022 19:22:30 +0100
Subject: [PATCH 26/56] fix linter

---
 espnet2/enh/separator/svoice_separator.py | 30 +++++++++++------------
 espnet2/tasks/enh.py                      |  4 ++-
 2 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/espnet2/enh/separator/svoice_separator.py b/espnet2/enh/separator/svoice_separator.py
index 5da45851a85..e1179b90e65 100644
--- a/espnet2/enh/separator/svoice_separator.py
+++ b/espnet2/enh/separator/svoice_separator.py
@@ -16,25 +16,25 @@
 def overlap_and_add(signal, frame_step):
     """Reconstructs a signal from a framed representation.
 
-    Adds potentially overlapping frames of a signal with shape
-    `[..., frames, frame_length]`, offsetting subsequent frames by `frame_step`.
-    The resulting tensor has shape `[..., output_size]` where
-        output_size = (frames - 1) * frame_step + frame_length
+        Adds potentially overlapping frames of a signal with shape
+        `[..., frames, frame_length]`, offsetting subsequent frames by `frame_step`.
+        The resulting tensor has shape `[..., output_size]` where
+            output_size = (frames - 1) * frame_step + frame_length
 
-    Args:
-        signal: A [..., frames, frame_length] Tensor. All dimensions may be unknown,
-            and rank must be at least 2.
-        frame_step: An integer denoting overlap offsets.
-            Must be less than or equal to frame_length.
+        Args:
+            signal: A [..., frames, frame_length] Tensor. All dimensions may be unknown,
+                and rank must be at least 2.
+            frame_step: An integer denoting overlap offsets.
+                Must be less than or equal to frame_length.
 
-    Returns:
-        A Tensor with shape [..., output_size] containing the
-            overlap-added frames of signal's inner-most two dimensions.
-        output_size = (frames - 1) * frame_step + frame_length
+        Returns:
+            A Tensor with shape [..., output_size] containing the
+                overlap-added frames of signal's inner-most two dimensions.
+            output_size = (frames - 1) * frame_step + frame_length
 
-    Based on
+        Based on
 
-https://github.com/tensorflow/tensorflow/blob/r1.12/tensorflow/contrib/signal/python/ops/reconstruction_ops.py
+    https://github.com/tensorflow/tensorflow/blob/r1.12/tensorflow/contrib/signal/python/ops/reconstruction_ops.py
     """
     outer_dimensions = signal.size()[:-2]
     frames, frame_length = signal.size()[-2:]
diff --git a/espnet2/tasks/enh.py b/espnet2/tasks/enh.py
index 8e086a46772..447731015bd 100644
--- a/espnet2/tasks/enh.py
+++ b/espnet2/tasks/enh.py
@@ -89,7 +89,9 @@
 
 loss_wrapper_choices = ClassChoices(
     name="loss_wrappers",
-    classes=dict(pit=PITSolver, fixed_order=FixedOrderSolver, multilayer_pit=MultiLayerPITSolver),
+    classes=dict(
+        pit=PITSolver, fixed_order=FixedOrderSolver, multilayer_pit=MultiLayerPITSolver
+    ),
     type_check=AbsLossWrapper,
     default=None,
 )

From 93336849a7f1066f84ada7275517ccd77b1be105 Mon Sep 17 00:00:00 2001
From: Zhaoheng Ni <zni@fb.com>
Date: Sun, 17 Apr 2022 08:43:36 +0100
Subject: [PATCH 27/56] reduce audio duration in test

---
 test/espnet2/enh/separator/test_svoice_separator.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/test/espnet2/enh/separator/test_svoice_separator.py b/test/espnet2/enh/separator/test_svoice_separator.py
index b5a7d39944d..b2fb191856c 100644
--- a/test/espnet2/enh/separator/test_svoice_separator.py
+++ b/test/espnet2/enh/separator/test_svoice_separator.py
@@ -15,7 +15,7 @@
 @pytest.mark.parametrize("segment_size", [2])
 @pytest.mark.parametrize("bidirectional", [False])
 @pytest.mark.parametrize("input_normalize", [False])
-def test_fasnet_separator_forward_backward_real(
+def test_svoice_separator_forward_backward(
     input_dim,
     enc_dim,
     kernel_size,
@@ -39,7 +39,7 @@ def test_fasnet_separator_forward_backward_real(
     )
     model.train()
 
-    x = torch.rand(2, 8000)
+    x = torch.rand(2, 800)
     x_lens = torch.tensor([400, 300], dtype=torch.long)
 
     separated, _, _ = model(x, ilens=x_lens)
@@ -50,8 +50,8 @@ def test_fasnet_separator_forward_backward_real(
     separated[0][0].mean().backward()
 
 
-def test_fasnet_separator_output_train():
-    x = torch.rand(2, 8000)
+def test_svoice_separator_output_train():
+    x = torch.rand(2, 800)
     x_lens = torch.tensor([10, 8], dtype=torch.long)
 
     for num_spk in range(1, 3):
@@ -73,8 +73,8 @@ def test_fasnet_separator_output_train():
         assert x[0].shape == waveforms[0][0][0].shape
 
 
-def test_fasnet_separator_output_eval():
-    x = torch.rand(2, 8000)
+def test_svoice_separator_output_eval():
+    x = torch.rand(2, 800)
     x_lens = torch.tensor([10, 8], dtype=torch.long)
 
     for num_spk in range(1, 3):

From 04d72770d52b5bc62ebfaefcee300841bde8932b Mon Sep 17 00:00:00 2001
From: Siddhant <siddhana@br011.ib.bridges2.psc.edu>
Date: Sun, 17 Apr 2022 13:49:53 -0400
Subject: [PATCH 28/56] Fix config and scoring

---
 egs2/slue-voxpopuli/asr1/conf/train_asr.yaml  |   2 +-
 ..._asr_conformer_lr2e-3_warmup5k_conv2d.yaml |  78 ------
 ...former_lr2e-3_warmup5k_hubert_conv2d2.yaml |  88 -------
 ...conformer_lr2e-3_warmup5k_tera_conv2d.yaml |  88 -------
 ...nformer_lr2e-3_warmup5k_vq_apc_conv2d.yaml |  88 -------
 ...rmer_lr2e-3_warmup5k_wav2vec2_conv2d2.yaml |  88 -------
 .../asr1/conf/tuning/train_asr_conformer.yaml |  65 -----
 .../conf/tuning/train_asr_conformer_nlu.yaml  |  77 ------
 .../tuning/train_asr_conformer_s3prl.yaml     |  85 -------
 .../train_asr_wav2vec2_conformer_small.yaml   |  90 -------
 .../asr1/local/data_prep_slue.py              |  66 -----
 egs2/slue-voxpopuli/asr1/local/f1_score.py    |  89 -------
 .../asr1/local/generate_asr_files.py          | 113 ---------
 egs2/slue-voxpopuli/asr1/local/score.py       | 233 +++++++++++++++++-
 14 files changed, 233 insertions(+), 1017 deletions(-)
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_conv2d.yaml
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_hubert_conv2d2.yaml
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_tera_conv2d.yaml
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_vq_apc_conv2d.yaml
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2.yaml
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer.yaml
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer_nlu.yaml
 delete mode 100755 egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer_s3prl.yaml
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_wav2vec2_conformer_small.yaml
 delete mode 100644 egs2/slue-voxpopuli/asr1/local/data_prep_slue.py
 delete mode 100755 egs2/slue-voxpopuli/asr1/local/f1_score.py
 delete mode 100644 egs2/slue-voxpopuli/asr1/local/generate_asr_files.py
 mode change 120000 => 100755 egs2/slue-voxpopuli/asr1/local/score.py

diff --git a/egs2/slue-voxpopuli/asr1/conf/train_asr.yaml b/egs2/slue-voxpopuli/asr1/conf/train_asr.yaml
index b79a2904e79..183f14e01f9 120000
--- a/egs2/slue-voxpopuli/asr1/conf/train_asr.yaml
+++ b/egs2/slue-voxpopuli/asr1/conf/train_asr.yaml
@@ -1 +1 @@
-tuning/train_asr_wav2vec2_conformer_small.yaml
\ No newline at end of file
+tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wavlm_conv2d2.yaml
\ No newline at end of file
diff --git a/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_conv2d.yaml b/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_conv2d.yaml
deleted file mode 100644
index 4f3a1ff1917..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_conv2d.yaml
+++ /dev/null
@@ -1,78 +0,0 @@
-specaug: specaug
-specaug_conf:
-    apply_time_warp: true
-    time_warp_window: 5
-    time_warp_mode: bicubic
-    apply_freq_mask: true
-    freq_mask_width_range:
-    - 0
-    - 27
-    num_freq_mask: 2
-    apply_time_mask: true
-    time_mask_width_ratio_range:
-    - 0.
-    - 0.05
-    num_time_mask: 2
-
-encoder: conformer
-encoder_conf:
-    output_size: 256
-    attention_heads: 4
-    linear_units: 1024
-    num_blocks: 12
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: conv2d
-    normalize_before: true
-    macaron_style: true
-    rel_pos_type: latest
-    pos_enc_layer_type: rel_pos
-    selfattention_layer_type: rel_selfattn
-    activation_type: swish
-    use_cnn_module: true
-    cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-    attention_heads: 4
-    linear_units: 2048
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    self_attention_dropout_rate: 0.1
-    src_attention_dropout_rate: 0.1
-
-model_conf:
-    ctc_weight: 0.3
-    lsm_weight: 0.1
-    length_normalized_loss: false
-    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
-
-frontend_conf:
-    n_fft: 512
-    win_length: 400
-    hop_length: 160
-
-seed: 2022
-log_interval: 100
-num_workers: 2
-batch_type: numel
-batch_bins: 6000000
-accum_grad: 2
-max_epoch: 70
-patience: none
-init: none
-best_model_criterion:
--   - valid
-    - acc
-    - max
-keep_nbest_models: 10
-
-optim: adam
-optim_conf:
-    lr: 0.002
-    weight_decay: 0.000001
-scheduler: warmuplr
-scheduler_conf:
-    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_hubert_conv2d2.yaml b/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_hubert_conv2d2.yaml
deleted file mode 100644
index 0de49db951b..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_hubert_conv2d2.yaml
+++ /dev/null
@@ -1,88 +0,0 @@
-specaug: specaug
-specaug_conf:
-    apply_time_warp: true
-    time_warp_window: 5
-    time_warp_mode: bicubic
-    apply_freq_mask: true
-    freq_mask_width_range:
-    - 0
-    - 27
-    num_freq_mask: 2
-    apply_time_mask: true
-    time_mask_width_ratio_range:
-    - 0.
-    - 0.05
-    num_time_mask: 2
-
-encoder: conformer
-encoder_conf:
-    output_size: 256
-    attention_heads: 4
-    linear_units: 1024
-    num_blocks: 12
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: conv2d2
-    normalize_before: true
-    macaron_style: true
-    rel_pos_type: latest
-    pos_enc_layer_type: rel_pos
-    selfattention_layer_type: rel_selfattn
-    activation_type: swish
-    use_cnn_module: true
-    cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-    attention_heads: 4
-    linear_units: 2048
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    self_attention_dropout_rate: 0.1
-    src_attention_dropout_rate: 0.1
-
-model_conf:
-    ctc_weight: 0.3
-    lsm_weight: 0.1
-    length_normalized_loss: false
-    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
-
-frontend: s3prl
-frontend_conf:
-    frontend_conf:
-        upstream: hubert_large_ll60k  # Note: If the upstream is changed, please change the input_size in the preencoder.
-    download_dir: ./hub
-    multilayer_feature: true
-
-preencoder: linear
-preencoder_conf:
-    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
-    output_size: 80
-
-freeze_param:
-    - "frontend.upstream"
-
-seed: 2022
-log_interval: 100
-num_workers: 2
-batch_type: numel
-batch_bins: 6000000
-accum_grad: 2
-max_epoch: 70
-patience: none
-init: none
-best_model_criterion:
--   - valid
-    - acc
-    - max
-keep_nbest_models: 10
-
-optim: adam
-optim_conf:
-    lr: 0.002
-    weight_decay: 0.000001
-scheduler: warmuplr
-scheduler_conf:
-    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_tera_conv2d.yaml b/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_tera_conv2d.yaml
deleted file mode 100644
index bde54149c47..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_tera_conv2d.yaml
+++ /dev/null
@@ -1,88 +0,0 @@
-specaug: specaug
-specaug_conf:
-    apply_time_warp: true
-    time_warp_window: 5
-    time_warp_mode: bicubic
-    apply_freq_mask: true
-    freq_mask_width_range:
-    - 0
-    - 27
-    num_freq_mask: 2
-    apply_time_mask: true
-    time_mask_width_ratio_range:
-    - 0.
-    - 0.05
-    num_time_mask: 2
-
-encoder: conformer
-encoder_conf:
-    output_size: 256
-    attention_heads: 4
-    linear_units: 1024
-    num_blocks: 12
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: conv2d
-    normalize_before: true
-    macaron_style: true
-    rel_pos_type: latest
-    pos_enc_layer_type: rel_pos
-    selfattention_layer_type: rel_selfattn
-    activation_type: swish
-    use_cnn_module: true
-    cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-    attention_heads: 4
-    linear_units: 2048
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    self_attention_dropout_rate: 0.1
-    src_attention_dropout_rate: 0.1
-
-model_conf:
-    ctc_weight: 0.3
-    lsm_weight: 0.1
-    length_normalized_loss: false
-    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
-
-frontend: s3prl
-frontend_conf:
-    frontend_conf:
-        upstream: tera_960hr  # Note: If the upstream is changed, please change the input_size in the preencoder.
-    download_dir: ./hub
-    multilayer_feature: true
-
-preencoder: linear
-preencoder_conf:
-    input_size: 768  # Note: If the upstream is changed, please change this value accordingly.
-    output_size: 80
-
-freeze_param:
-    - "frontend.upstream"
-
-seed: 2022
-log_interval: 100
-num_workers: 2
-batch_type: numel
-batch_bins: 6000000
-accum_grad: 2
-max_epoch: 70
-patience: none
-init: none
-best_model_criterion:
--   - valid
-    - acc
-    - max
-keep_nbest_models: 10
-
-optim: adam
-optim_conf:
-    lr: 0.002
-    weight_decay: 0.000001
-scheduler: warmuplr
-scheduler_conf:
-    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_vq_apc_conv2d.yaml b/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_vq_apc_conv2d.yaml
deleted file mode 100644
index 0d17cd8cef0..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_vq_apc_conv2d.yaml
+++ /dev/null
@@ -1,88 +0,0 @@
-specaug: specaug
-specaug_conf:
-    apply_time_warp: true
-    time_warp_window: 5
-    time_warp_mode: bicubic
-    apply_freq_mask: true
-    freq_mask_width_range:
-    - 0
-    - 27
-    num_freq_mask: 2
-    apply_time_mask: true
-    time_mask_width_ratio_range:
-    - 0.
-    - 0.05
-    num_time_mask: 2
-
-encoder: conformer
-encoder_conf:
-    output_size: 256
-    attention_heads: 4
-    linear_units: 1024
-    num_blocks: 12
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: conv2d
-    normalize_before: true
-    macaron_style: true
-    rel_pos_type: latest
-    pos_enc_layer_type: rel_pos
-    selfattention_layer_type: rel_selfattn
-    activation_type: swish
-    use_cnn_module: true
-    cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-    attention_heads: 4
-    linear_units: 2048
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    self_attention_dropout_rate: 0.1
-    src_attention_dropout_rate: 0.1
-
-model_conf:
-    ctc_weight: 0.3
-    lsm_weight: 0.1
-    length_normalized_loss: false
-    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
-
-frontend: s3prl
-frontend_conf:
-    frontend_conf:
-        upstream: vq_apc_960hr  # Note: If the upstream is changed, please change the input_size in the preencoder.
-    download_dir: ./hub
-    multilayer_feature: true
-
-preencoder: linear
-preencoder_conf:
-    input_size: 512  # Note: If the upstream is changed, please change this value accordingly.
-    output_size: 80
-
-freeze_param:
-    - "frontend.upstream"
-
-seed: 2022
-log_interval: 100
-num_workers: 2
-batch_type: numel
-batch_bins: 6000000
-accum_grad: 2
-max_epoch: 70
-patience: none
-init: none
-best_model_criterion:
--   - valid
-    - acc
-    - max
-keep_nbest_models: 10
-
-optim: adam
-optim_conf:
-    lr: 0.002
-    weight_decay: 0.000001
-scheduler: warmuplr
-scheduler_conf:
-    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2.yaml b/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2.yaml
deleted file mode 100644
index eb25f604f1e..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2.yaml
+++ /dev/null
@@ -1,88 +0,0 @@
-specaug: specaug
-specaug_conf:
-    apply_time_warp: true
-    time_warp_window: 5
-    time_warp_mode: bicubic
-    apply_freq_mask: true
-    freq_mask_width_range:
-    - 0
-    - 27
-    num_freq_mask: 2
-    apply_time_mask: true
-    time_mask_width_ratio_range:
-    - 0.
-    - 0.05
-    num_time_mask: 2
-
-encoder: conformer
-encoder_conf:
-    output_size: 256
-    attention_heads: 4
-    linear_units: 1024
-    num_blocks: 12
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: conv2d2
-    normalize_before: true
-    macaron_style: true
-    rel_pos_type: latest
-    pos_enc_layer_type: rel_pos
-    selfattention_layer_type: rel_selfattn
-    activation_type: swish
-    use_cnn_module: true
-    cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-    attention_heads: 4
-    linear_units: 2048
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    self_attention_dropout_rate: 0.1
-    src_attention_dropout_rate: 0.1
-
-model_conf:
-    ctc_weight: 0.3
-    lsm_weight: 0.1
-    length_normalized_loss: false
-    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
-
-frontend: s3prl
-frontend_conf:
-    frontend_conf:
-        upstream: wav2vec2_large_ll60k  # Note: If the upstream is changed, please change the input_size in the preencoder.
-    download_dir: ./hub
-    multilayer_feature: true
-
-preencoder: linear
-preencoder_conf:
-    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
-    output_size: 80
-
-freeze_param:
-    - "frontend.upstream"
-
-seed: 2022
-log_interval: 100
-num_workers: 2
-batch_type: numel
-batch_bins: 6000000
-accum_grad: 2
-max_epoch: 70
-patience: none
-init: none
-best_model_criterion:
--   - valid
-    - acc
-    - max
-keep_nbest_models: 10
-
-optim: adam
-optim_conf:
-    lr: 0.002
-    weight_decay: 0.000001
-scheduler: warmuplr
-scheduler_conf:
-    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer.yaml
deleted file mode 100644
index ee983ed0e8c..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer.yaml
+++ /dev/null
@@ -1,65 +0,0 @@
-# network architecture
-# encoder related
-encoder: conformer
-encoder_conf:
-    output_size: 512
-    attention_heads: 8
-    linear_units: 2048
-    num_blocks: 12
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: conv2d
-    normalize_before: true
-    macaron_style: true
-    pos_enc_layer_type: "rel_pos"
-    selfattention_layer_type: "rel_selfattn"
-    activation_type: "swish"
-    use_cnn_module:  true
-    cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-    attention_heads: 8
-    linear_units: 2048
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    self_attention_dropout_rate: 0.1
-    src_attention_dropout_rate: 0.1
-
-optim: adam
-optim_conf:
-    lr: 0.0002
-scheduler: warmuplr     # pytorch v1.1.0+ required #Tune warmup steps
-scheduler_conf:
-    warmup_steps: 25000
-max_epoch: 50
-
-model_conf:
-    ctc_weight: 0.3
-    lsm_weight: 0.1
-    length_normalized_loss: false
-    extract_feats_in_collect_stats: false   # Note: "False" means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend.
-
-specaug: specaug
-specaug_conf:
-    apply_time_warp: true
-    time_warp_window: 5
-    time_warp_mode: bicubic
-    apply_freq_mask: true
-    freq_mask_width_range:
-    - 0
-    - 30
-    num_freq_mask: 2
-    apply_time_mask: true
-    time_mask_width_range:
-    - 0
-    - 40
-    num_time_mask: 2
-
-best_model_criterion:
--   - valid
-    - acc
-    - max
-keep_nbest_models: 10
\ No newline at end of file
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer_nlu.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer_nlu.yaml
deleted file mode 100644
index a10d58445aa..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer_nlu.yaml
+++ /dev/null
@@ -1,77 +0,0 @@
-# network architecture
-# encoder related
-encoder: conformer
-encoder_conf:
-    output_size: 512
-    attention_heads: 8
-    linear_units: 2048
-    num_blocks: 12
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: conv2d
-    normalize_before: true
-    macaron_style: true
-    pos_enc_layer_type: "rel_pos"
-    selfattention_layer_type: "rel_selfattn"
-    activation_type: "swish"
-    use_cnn_module:  true
-    cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-    attention_heads: 8
-    linear_units: 2048
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    self_attention_dropout_rate: 0.1
-    src_attention_dropout_rate: 0.1
-
-postencoder: hugging_face_transformers
-postencoder_conf:
-    # pick up a model from https://huggingface.co/models?filter=transformers
-    # most of models should work, but maybe some don't
-    # known to work: bert, gpt2, xlnet, roberta, mpnet, t5, bart
-    # xlnet currently works for single gpu only
-    model_name_or_path: "bert-base-uncased"
-
-freeze_param: [
-"postencoder.transformer"
-]
-
-optim: adam
-optim_conf:
-    lr: 0.0002
-scheduler: warmuplr     # pytorch v1.1.0+ required #Tune warmup steps
-scheduler_conf:
-    warmup_steps: 25000
-max_epoch: 50
-
-model_conf:
-    ctc_weight: 0.3
-    lsm_weight: 0.1
-    length_normalized_loss: false
-    extract_feats_in_collect_stats: false   # Note: "False" means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend.
-
-specaug: specaug
-specaug_conf:
-    apply_time_warp: true
-    time_warp_window: 5
-    time_warp_mode: bicubic
-    apply_freq_mask: true
-    freq_mask_width_range:
-    - 0
-    - 30
-    num_freq_mask: 2
-    apply_time_mask: true
-    time_mask_width_range:
-    - 0
-    - 40
-    num_time_mask: 2
-
-best_model_criterion:
--   - valid
-    - acc
-    - max
-keep_nbest_models: 10
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer_s3prl.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer_s3prl.yaml
deleted file mode 100755
index c6acbbb652a..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_conformer_s3prl.yaml
+++ /dev/null
@@ -1,85 +0,0 @@
-# network architecture
-# encoder related
-encoder: conformer
-encoder_conf:
-    output_size: 512
-    attention_heads: 8
-    linear_units: 2048
-    num_blocks: 12
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: conv2d
-    normalize_before: true
-    macaron_style: true
-    pos_enc_layer_type: "rel_pos"
-    selfattention_layer_type: "rel_selfattn"
-    activation_type: "swish"
-    use_cnn_module:  true
-    cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-    attention_heads: 8
-    linear_units: 2048
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    self_attention_dropout_rate: 0.1
-    src_attention_dropout_rate: 0.1
-
-optim: adam
-optim_conf:
-    lr: 0.0002
-scheduler: warmuplr     # pytorch v1.1.0+ required
-scheduler_conf:
-    warmup_steps: 25000
-max_epoch: 50
-
-freeze_param: [
-"frontend.upstream"
-]
-
-frontend_conf:
-  n_fft: 512
-  hop_length: 256
-
-frontend: s3prl
-frontend_conf:
-    frontend_conf:
-        upstream: hubert_large_ll60k  # Note: If the upstream is changed, please change the input_size in the preencoder.
-    download_dir: ./hub
-    multilayer_feature: True
-
-preencoder: linear
-preencoder_conf:
-    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
-    output_size: 80
-
-model_conf:
-    ctc_weight: 0.3
-    lsm_weight: 0.1
-    length_normalized_loss: false
-    extract_feats_in_collect_stats: false   # Note: "False" means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend.
-
-specaug: specaug
-specaug_conf:
-    apply_time_warp: true
-    time_warp_window: 5
-    time_warp_mode: bicubic
-    apply_freq_mask: true
-    freq_mask_width_range:
-    - 0
-    - 30
-    num_freq_mask: 2
-    apply_time_mask: true
-    time_mask_width_range:
-    - 0
-    - 40
-    num_time_mask: 2
-
-best_model_criterion:
--   - valid
-    - acc
-    - max
-keep_nbest_models: 10
\ No newline at end of file
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_wav2vec2_conformer_small.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_wav2vec2_conformer_small.yaml
deleted file mode 100644
index fe8f902cb63..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/tuning/train_asr_wav2vec2_conformer_small.yaml
+++ /dev/null
@@ -1,90 +0,0 @@
-# network architecture
-# encoder related
-batch_type: numel
-batch_bins: 6000000
-encoder: conformer
-accum_grad: 2
-
-encoder_conf:
-    output_size: 256
-    attention_heads: 4
-    linear_units: 1024
-    num_blocks: 12
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: conv2d2
-    normalize_before: true
-    macaron_style: true
-    pos_enc_layer_type: "rel_pos"
-    selfattention_layer_type: "rel_selfattn"
-    activation_type: "swish"
-    use_cnn_module:  true
-    cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-    attention_heads: 4
-    linear_units: 2048
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    self_attention_dropout_rate: 0.1
-    src_attention_dropout_rate: 0.1
-
-optim: adam
-optim_conf:
-    lr: 0.002
-    weight_decay: 1.0e-06
-scheduler: warmuplr     # pytorch v1.1.0+ required #Tune warmup steps
-scheduler_conf:
-    warmup_steps: 5000
-max_epoch: 100
-
-freeze_param: [
-"frontend.upstream"
-]
-
-frontend_conf:
-  n_fft: 512
-  hop_length: 256
-
-frontend: s3prl
-frontend_conf:
-    frontend_conf:
-        upstream: wav2vec2_large_ll60k  # Note: If the upstream is changed, please change the input_size in the preencoder.
-    download_dir: ./hub
-    multilayer_feature: True
-
-preencoder: linear
-preencoder_conf:
-    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
-    output_size: 80
-
-model_conf:
-    ctc_weight: 0.3
-    lsm_weight: 0.1
-    length_normalized_loss: false
-    extract_feats_in_collect_stats: false   # Note: "False" means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend.
-
-specaug: specaug
-specaug_conf:
-    apply_time_warp: true
-    time_warp_window: 5
-    time_warp_mode: bicubic
-    apply_freq_mask: true
-    freq_mask_width_range:
-    - 0
-    - 30
-    num_freq_mask: 2
-    apply_time_mask: true
-    time_mask_width_range:
-    - 0
-    - 40
-    num_time_mask: 2
-
-best_model_criterion:
--   - valid
-    - acc
-    - max
-keep_nbest_models: 10
diff --git a/egs2/slue-voxpopuli/asr1/local/data_prep_slue.py b/egs2/slue-voxpopuli/asr1/local/data_prep_slue.py
deleted file mode 100644
index 89b42059e30..00000000000
--- a/egs2/slue-voxpopuli/asr1/local/data_prep_slue.py
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2021  Siddhant Arora
-#           2021  Carnegie Mellon University
-# Apache 2.0
-
-
-import os
-import re
-import sys
-import pandas as pd
-
-if len(sys.argv) != 2:
-    print("Usage: python data_prep.py [root]")
-    sys.exit(1)
-root = sys.argv[1]
-
-dir_dict = {
-    "train": "slue-voxceleb_fine-tune.tsv",
-    "devel": "slue-voxceleb_dev.tsv",
-    "test": "slue-voxceleb_test_blind.tsv",
-}
-
-for x in dir_dict:
-    with open(os.path.join("data", x, "text"), "w") as text_f, open(
-        os.path.join("data", x, "wav.scp"), "w"
-    ) as wav_scp_f, open(
-        os.path.join("data", x, "transcript"), "w"
-    ) as transcript_f, open(
-        os.path.join("data", x, "utt2spk"), "w"
-    ) as utt2spk_f:
-
-        text_f.truncate()
-        wav_scp_f.truncate()
-        utt2spk_f.truncate()
-        transcript_df = pd.read_csv(os.path.join(root, dir_dict[x]), sep="\t")
-        # lines = sorted(transcript_df.values, key=lambda s: s[0])
-        for row in transcript_df.values:
-            if x == "test":
-                speaker = row[1]
-                words = (
-                    "<blank>"  # Test set is blind, will have to submit to leaderboard
-                )
-            else:
-                if row[4] == "<mixed>":
-                    continue
-                print(x)
-                print(row)
-                words = (
-                    row[4].replace(" ", "_")
-                    + " "
-                    + row[1].encode("ascii", "ignore").decode()
-                )
-                print(words)
-                speaker = row[2]
-            if x == "train":
-                path = "fine-tune_raw/" + row[0] + ".flac"
-            elif x == "devel":
-                path = "dev_raw/" + row[0] + ".flac"
-            else:
-                path = "test_raw/" + row[0] + ".flac"
-            utt_id = row[0]
-            # print(utt_id + " " + words + "\n")
-            text_f.write(utt_id + " " + words + "\n")
-            wav_scp_f.write(utt_id + " " + root + "/" + path + "\n")
-            utt2spk_f.write(utt_id + " " + speaker + "\n")
diff --git a/egs2/slue-voxpopuli/asr1/local/f1_score.py b/egs2/slue-voxpopuli/asr1/local/f1_score.py
deleted file mode 100755
index 4f45752a812..00000000000
--- a/egs2/slue-voxpopuli/asr1/local/f1_score.py
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2021  Siddhant Arora
-#           2021  Carnegie Mellon University
-# Apache 2.0
-
-
-import os
-import re
-import sys
-import pandas as pd
-import argparse
-from sklearn.metrics import f1_score
-from sklearn.metrics import classification_report
-
-
-def get_classification_result(hyp_file, ref_file):
-    hyp_lines = [line for line in hyp_file]
-    ref_lines = [line for line in ref_file]
-
-    error = 0
-    hyp_intent_arr = []
-    ref_intent_arr = []
-    for line_count in range(len(hyp_lines)):
-        hyp_intent = hyp_lines[line_count].split(" ")[0]
-        ref_intent = ref_lines[line_count].split(" ")[0]
-        hyp_intent_arr.append(hyp_intent)
-        ref_intent_arr.append(ref_intent)
-    print(classification_report(ref_intent_arr, hyp_intent_arr))
-    return f1_score(ref_intent_arr, hyp_intent_arr, average="macro")
-
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--exp_root", required=True, help="Directory to save experiments")
-parser.add_argument(
-    "--valid_folder",
-    default="inference_asr_model_valid.acc.ave_10best/devel/",
-    help="Directory inside exp_root containing inference on valid set",
-)
-parser.add_argument(
-    "--test_folder",
-    default="inference_asr_model_valid.acc.ave_10best/test/",
-    help="Directory inside exp_root containing inference on test set",
-)
-parser.add_argument(
-    "--utterance_test_folder",
-    default=None,
-    help="Directory inside exp_root containing inference on utterance test set",
-)
-
-args = parser.parse_args()
-
-exp_root = args.exp_root
-valid_inference_folder = args.valid_folder
-test_inference_folder = args.test_folder
-
-valid_hyp_file = open(
-    os.path.join(exp_root, valid_inference_folder + "score_wer/hyp.trn")
-)
-valid_ref_file = open(
-    os.path.join(exp_root, valid_inference_folder + "score_wer/ref.trn")
-)
-
-result = get_classification_result(valid_hyp_file, valid_ref_file)
-print("Valid Macro F1")
-print(result)
-
-test_hyp_file = open(
-    os.path.join(exp_root, test_inference_folder + "score_wer/hyp.trn")
-)
-test_ref_file = open(
-    os.path.join(exp_root, test_inference_folder + "score_wer/ref.trn")
-)
-
-result = get_classification_result(test_hyp_file, test_ref_file)
-print("Test Intent Macro F1")
-print(result)
-
-if args.utterance_test_folder is not None:
-    utt_test_inference_folder = args.utterance_test_folder
-    utt_test_hyp_file = open(
-        os.path.join(exp_root, utt_test_inference_folder + "score_wer/hyp.trn")
-    )
-    utt_test_ref_file = open(
-        os.path.join(exp_root, utt_test_inference_folder + "score_wer/ref.trn")
-    )
-    result = get_classification_result(utt_test_hyp_file, utt_test_ref_file)
-    print("Unseen Utterance Test Macro F1")
-    print(result)
diff --git a/egs2/slue-voxpopuli/asr1/local/generate_asr_files.py b/egs2/slue-voxpopuli/asr1/local/generate_asr_files.py
deleted file mode 100644
index dd8a4645410..00000000000
--- a/egs2/slue-voxpopuli/asr1/local/generate_asr_files.py
+++ /dev/null
@@ -1,113 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2021  Siddhant Arora
-#           2021  Carnegie Mellon University
-# Apache 2.0
-
-
-import os
-import re
-import sys
-import pandas as pd
-import argparse
-
-
-def generate_asr_files(txt_file, transcript_file):
-    line_arr = [line for line in txt_file]
-    for line in line_arr:
-        if len(line.split("\t")) > 2:
-            print(line)
-            exit()
-        if len(line.split("\t")[0].split()) == 1:
-            text = "<blank>"
-        else:
-            text = line.split("\t")[0].split()[1].replace("▁", "")
-        for sub_word in line.split("\t")[0].split()[2:]:
-            if "▁" in sub_word:
-                text = text + " " + sub_word.replace("▁", "")
-            else:
-                text = text + sub_word
-        if len(text) == 0:
-            text = "<blank>"
-        wav_name = line.split("\t")[1]
-        transcript_file.write(text + "\t" + wav_name)
-
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--exp_root", required=True, help="Directory to save experiments")
-parser.add_argument(
-    "--valid_folder",
-    default="inference_asr_model_valid.acc.ave_10best/devel/",
-    help="Directory inside exp_root containing inference on valid set",
-)
-parser.add_argument(
-    "--test_folder",
-    default="inference_asr_model_valid.acc.ave_10best/test/",
-    help="Directory inside exp_root containing inference on test set",
-)
-parser.add_argument(
-    "--utterance_test_folder",
-    default=None,
-    help="Directory inside exp_root containing inference on utterance test set",
-)
-
-args = parser.parse_args()
-
-exp_root = args.exp_root
-valid_inference_folder = args.valid_folder
-test_inference_folder = args.test_folder
-
-valid_hyp_file = open(
-    os.path.join(exp_root, valid_inference_folder + "score_wer/hyp.trn")
-)
-valid_ref_file = open(
-    os.path.join(exp_root, valid_inference_folder + "score_wer/ref.trn")
-)
-
-valid_hyp_write_file = open(
-    os.path.join(exp_root, valid_inference_folder + "score_wer/hyp_asr.trn"), "w"
-)
-valid_ref_write_file = open(
-    os.path.join(exp_root, valid_inference_folder + "score_wer/ref_asr.trn"), "w"
-)
-
-generate_asr_files(valid_hyp_file, valid_hyp_write_file)
-
-generate_asr_files(valid_ref_file, valid_ref_write_file)
-
-
-test_hyp_file = open(
-    os.path.join(exp_root, test_inference_folder + "score_wer/hyp.trn")
-)
-test_ref_file = open(
-    os.path.join(exp_root, test_inference_folder + "score_wer/ref.trn")
-)
-test_hyp_write_file = open(
-    os.path.join(exp_root, test_inference_folder + "score_wer/hyp_asr.trn"), "w"
-)
-test_ref_write_file = open(
-    os.path.join(exp_root, test_inference_folder + "score_wer/ref_asr.trn"), "w"
-)
-
-generate_asr_files(test_hyp_file, test_hyp_write_file)
-
-generate_asr_files(test_ref_file, test_ref_write_file)
-
-
-if args.utterance_test_folder is not None:
-    utt_test_inference_folder = args.utterance_test_folder
-    utt_test_hyp_file = open(
-        os.path.join(exp_root, utt_test_inference_folder + "score_wer/hyp.trn")
-    )
-    utt_test_ref_file = open(
-        os.path.join(exp_root, utt_test_inference_folder + "score_wer/ref.trn")
-    )
-    utt_test_hyp_write_file = open(
-        os.path.join(exp_root, utt_test_inference_folder + "score_wer/hyp_asr.trn"), "w"
-    )
-    utt_test_ref_write_file = open(
-        os.path.join(exp_root, utt_test_inference_folder + "score_wer/ref_asr.trn"), "w"
-    )
-    generate_asr_files(utt_test_hyp_file, utt_test_hyp_write_file)
-
-    generate_asr_files(utt_test_ref_file, utt_test_ref_write_file)
diff --git a/egs2/slue-voxpopuli/asr1/local/score.py b/egs2/slue-voxpopuli/asr1/local/score.py
deleted file mode 120000
index 32c1318cf74..00000000000
--- a/egs2/slue-voxpopuli/asr1/local/score.py
+++ /dev/null
@@ -1 +0,0 @@
-../../../TEMPLATE/asr1/pyscripts/utils/score_intent.py
\ No newline at end of file
diff --git a/egs2/slue-voxpopuli/asr1/local/score.py b/egs2/slue-voxpopuli/asr1/local/score.py
new file mode 100755
index 00000000000..6f95b744910
--- /dev/null
+++ b/egs2/slue-voxpopuli/asr1/local/score.py
@@ -0,0 +1,232 @@
+#!/usr/bin/env bash
+
+# Copyright 2021  Siddhant Arora, Yifan Peng
+#           2021  Carnegie Mellon University
+# Apache 2.0
+
+import json
+import os
+import re
+import sys
+import pandas as pd
+import argparse
+
+import eval_utils
+
+ontonotes_to_combined_label = {
+    "GPE": "PLACE",
+    "LOC": "PLACE",
+    "CARDINAL": "QUANT",
+    "MONEY": "QUANT",
+    "ORDINAL": "QUANT",
+    "PERCENT": "QUANT",
+    "QUANTITY": "QUANT",
+    "ORG": "ORG",
+    "DATE": "WHEN",
+    "TIME": "WHEN",
+    "NORP": "NORP",
+    "PERSON": "PERSON",
+    "LAW": "LAW",
+}
+combined_labels = set(ontonotes_to_combined_label.values())
+special_tokens = list(combined_labels) + ["FILL", "SEP"]
+
+
+def preprocess_sentence(line):
+    # Ensure special tokens are never merged with others
+    for label in special_tokens:
+        line = line.replace(label, "▁" + label + "▁")
+
+    line = line.strip().replace(" ", "").replace("▁", " ")
+    line = re.sub(" +", " ", line).strip()  # remove consecutive spaces
+
+    # Also return the sentence without special tokens
+    valid_tokens = []
+    for token in line.split():
+        if token not in special_tokens:
+            valid_tokens.append(token)
+    valid_line = " ".join(valid_tokens)
+    valid_line = valid_line.replace(" 's", "'s")  # combine 's with the previous word
+
+    return line, valid_line
+
+
+def make_distinct(label_lst):
+    """
+    Make the label_lst distinct
+    """
+    tag2cnt, new_tag_lst = {}, []
+    if len(label_lst) > 0:
+        for tag_item in label_lst:
+            _ = tag2cnt.setdefault(tag_item, 0)
+            tag2cnt[tag_item] += 1
+            tag, wrd = tag_item
+            new_tag_lst.append((tag, wrd, tag2cnt[tag_item]))
+        assert len(new_tag_lst) == len(set(new_tag_lst))
+    return new_tag_lst
+
+
+def process_line(line, label_F1=False):
+    label_lst = []
+    line = line.replace("  ", " ")
+    wrd_lst = line.split(" ")
+    phrase_lst, is_entity, num_illegal_assigments = [], False, 0
+    for idx, wrd in enumerate(wrd_lst):
+        if wrd in combined_labels:
+            if is_entity:
+                phrase_lst = []
+                num_illegal_assigments += 1
+            is_entity = True
+            entity_tag = wrd
+        elif wrd == "SEP":
+            if is_entity:
+                if len(phrase_lst) > 0:
+                    phrase_lst.remove("FILL")
+                    if label_F1 == True:
+                        label_lst.append((entity_tag, "phrase"))
+                    else:
+                        label_lst.append((entity_tag, " ".join(phrase_lst)))
+                else:
+                    num_illegal_assignments += 1
+                phrase_lst = []
+                is_entity = False
+            else:
+                num_illegal_assigments += 1
+
+        else:
+            if is_entity:
+                phrase_lst.append(wrd)
+
+    return make_distinct(label_lst)
+
+
+def get_classification_result(hyp_file, ref_file, hyp_asr_file, ref_asr_file):
+    hyp_lines = [line for line in hyp_file]
+    ref_lines = [line for line in ref_file]
+
+    hyp_list = []
+    ref_list = []
+
+    hyp_asr_list = []
+    ref_asr_list = []
+
+    hyp_label_list = []
+    ref_label_list = []
+
+    for line_count in range(len(hyp_lines)):
+        hyp_tokens = hyp_lines[line_count].split()
+        ref_tokens = ref_lines[line_count].split()
+
+        # The last "word" is utt_id
+        hyp_id = hyp_tokens[-1]
+        ref_id = ref_tokens[-1]
+        assert hyp_id == ref_id, f"hyp_id: {hyp_id}, ref_id: {ref_id}"
+
+        # Remove utt_id
+        hyp_line = " ".join(hyp_tokens[:-1])
+        ref_line = " ".join(ref_tokens[:-1])
+
+        # De-tokenize
+        hyp_line, hyp_line_asr = preprocess_sentence(hyp_line)
+        ref_line, ref_line_asr = preprocess_sentence(ref_line)
+
+        # Save results for computing F1
+        hyp_list.append(process_line(hyp_line))
+        ref_list.append(process_line(ref_line))
+
+        # Pure ASR text without special tokens
+        hyp_asr_list.append(hyp_line_asr + "\t" + hyp_id)
+        ref_asr_list.append(ref_line_asr + "\t" + ref_id)
+
+        # Save results for computing label-F1
+        hyp_label_list.append(process_line(hyp_line, label_F1=True))
+        ref_label_list.append(process_line(ref_line, label_F1=True))
+
+    # NER F1 score
+    metrics = eval_utils.get_ner_scores(hyp_list, ref_list)
+
+    # Write ASR text for computing WER later
+    for ln in hyp_asr_list:
+        hyp_asr_file.write(ln + "\n")
+    for ln in ref_asr_list:
+        ref_asr_file.write(ln + "\n")
+
+    # NER label-F1 score
+    label_metrics = eval_utils.get_ner_scores(hyp_label_list, ref_label_list)
+
+    return metrics, label_metrics
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--exp_root", required=True, help="Directory to save experiments")
+parser.add_argument(
+    "--valid_folder",
+    default="decode_asr_asr_model_valid.acc.ave/devel/",
+    help="Directory inside exp_root containing inference on valid set",
+)
+parser.add_argument(
+    "--test_folder",
+    default="decode_asr_asr_model_valid.acc.ave/test/",
+    help="Directory inside exp_root containing inference on test set",
+)
+args = parser.parse_args()
+
+exp_root = args.exp_root
+valid_inference_folder = args.valid_folder
+test_inference_folder = args.test_folder
+
+# Read original tokenized text
+valid_hyp_file = open(
+    os.path.join(exp_root, valid_inference_folder + "score_ter/hyp.trn")
+)
+valid_ref_file = open(
+    os.path.join(exp_root, valid_inference_folder + "score_ter/ref.trn")
+)
+
+# Write detokenized text
+valid_hyp_asr_file = open(
+    os.path.join(exp_root, valid_inference_folder + "score_ter/hyp_asr.trn"), "w"
+)
+valid_ref_asr_file = open(
+    os.path.join(exp_root, valid_inference_folder + "score_ter/ref_asr.trn"), "w"
+)
+
+result, label_result = get_classification_result(
+    valid_hyp_file, valid_ref_file, valid_hyp_asr_file, valid_ref_asr_file
+)
+print("Valid F1:")
+print(json.dumps(result, indent=4))
+print()
+print("Valid label-F1:")
+print(json.dumps(label_result, indent=4))
+print()
+
+
+if os.path.isdir(test_inference_folder):
+    # Read files
+    test_hyp_file = open(
+        os.path.join(exp_root, test_inference_folder + "score_ter/hyp.trn")
+    )
+    test_ref_file = open(
+        os.path.join(exp_root, test_inference_folder + "score_ter/ref.trn")
+    )
+
+    # Write files
+    test_hyp_asr_file = open(
+        os.path.join(exp_root, test_inference_folder + "score_ter/hyp_asr.trn"), "w"
+    )
+    test_ref_asr_file = open(
+        os.path.join(exp_root, test_inference_folder + "score_ter/ref_asr.trn"), "w"
+    )
+
+    result, label_result = get_classification_result(
+        test_hyp_file, test_ref_file, test_hyp_asr_file, test_ref_asr_file
+    )
+    print("Test F1:")
+    print(json.dumps(result, indent=4))
+    print()
+    print("Test label-F1:")
+    print(json.dumps(label_result, indent=4))
+    print()
+else:
+    print("[Warning] Skip F1 and label-F1 on test set as it does not exist.\n")

From d63bbbd0fad70ae68825640ecce47888d4bafff6 Mon Sep 17 00:00:00 2001
From: Siddhant <siddhana@br013.ib.bridges2.psc.edu>
Date: Sun, 17 Apr 2022 15:11:11 -0400
Subject: [PATCH 29/56] Fix CI errors

---
 egs2/slue-voxpopuli/asr1/local/eval_utils.py | 9 ++++++---
 egs2/slue-voxpopuli/asr1/local/score.py      | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/egs2/slue-voxpopuli/asr1/local/eval_utils.py b/egs2/slue-voxpopuli/asr1/local/eval_utils.py
index fc5c3f9c473..9310735eb56 100644
--- a/egs2/slue-voxpopuli/asr1/local/eval_utils.py
+++ b/egs2/slue-voxpopuli/asr1/local/eval_utils.py
@@ -6,18 +6,21 @@
 
 def get_ner_scores(all_gt, all_predictions):
     """
-    Evalutes per-label and overall (micro and macro) metrics of precision, recall, and fscore
+    Evalutes per-label and overall (micro and macro) metrics
+    of precision, recall, and fscore
     Input:
         all_gt/all_predictions:
             List of list of tuples: (label, phrase, identifier)
             Each list of tuples correspond to a sentence:
                 label: entity tag
                 phrase: entity phrase
-                tuple_identifier: identifier to differentiate repeating (label, phrase) pairs
+                tuple_identifier: identifier to differentiate
+                repeating (label, phrase) pairs
     Returns:
         Dictionary of metrics
     Example:
-        List of GT (label, phrase) pairs of a sentence: [(GPE, "eu"), (DATE, "today"), (GPE, "eu")]
+        List of GT (label, phrase) pairs of a sentence:
+        [(GPE, "eu"), (DATE, "today"), (GPE, "eu")]
         all_gt: [(GPE, "eu", 0), (DATE, "today", 0), (GPE, "eu", 1)]
     """
     metrics = {}
diff --git a/egs2/slue-voxpopuli/asr1/local/score.py b/egs2/slue-voxpopuli/asr1/local/score.py
index 6f95b744910..4239ed2b151 100755
--- a/egs2/slue-voxpopuli/asr1/local/score.py
+++ b/egs2/slue-voxpopuli/asr1/local/score.py
@@ -82,7 +82,7 @@ def process_line(line, label_F1=False):
             if is_entity:
                 if len(phrase_lst) > 0:
                     phrase_lst.remove("FILL")
-                    if label_F1 == True:
+                    if label_F1 is True:
                         label_lst.append((entity_tag, "phrase"))
                     else:
                         label_lst.append((entity_tag, " ".join(phrase_lst)))

From 063a985f25fc035c674a070544b3712be307b761 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Wed, 20 Apr 2022 15:56:29 +0800
Subject: [PATCH 30/56] Fix typos and add support for inference_enh_config in
 enh_asr and enh_st tasks

---
 egs2/TEMPLATE/enh_asr1/enh_asr.sh | 33 +++++++++++++++++---------
 egs2/TEMPLATE/enh_st1/enh_st.sh   | 39 +++++++++++++++++++------------
 2 files changed, 46 insertions(+), 26 deletions(-)

diff --git a/egs2/TEMPLATE/enh_asr1/enh_asr.sh b/egs2/TEMPLATE/enh_asr1/enh_asr.sh
index 3d238938a57..7ca4f9ddc77 100755
--- a/egs2/TEMPLATE/enh_asr1/enh_asr.sh
+++ b/egs2/TEMPLATE/enh_asr1/enh_asr.sh
@@ -122,6 +122,8 @@ dereverb_ref_num=1
 # Evaluation related
 scoring_protocol="STOI SDR SAR SIR SI_SNR"
 ref_channel=0
+inference_enh_tag=      # Prefix to the result dir for ENH inference.
+inference_enh_config=   # Config for enhancement.
 
 # Enh Training data related
 use_dereverb_ref=false
@@ -453,6 +455,14 @@ if [ -z "${inference_tag}" ]; then
     fi
 fi
 
+if [ -z "${inference_enh_tag}" ]; then
+    if [ -n "${inference_enh_config}" ]; then
+        inference_enh_tag="$(basename "${inference_enh_config}" .yaml)"
+    else
+        inference_enh_tag=enhanced
+    fi
+fi
+
 # ========================== Main stages start from here. ==========================
 
 if ! "${skip_data_prep}"; then
@@ -518,7 +528,7 @@ if ! "${skip_data_prep}"; then
                 expand_utt_extra_files=""
                 for extra_file in ${utt_extra_files}; do
                     # with regex to suuport multi-references
-                    for single_file in $(ls data/"${dset}"/${extra_file}*); do
+                    for single_file in "data/${dset}/${extra_file}"*; do
                         cp ${single_file} "${data_feats}${_suf}/${dset}"
                         expand_utt_extra_files="${expand_utt_extra_files} $(basename ${single_file})"
                     done
@@ -1293,12 +1303,12 @@ if ! "${skip_eval}"; then
         _opts=
 
         # 2. Generate run.sh
-        log "Generate '${enh_asr_exp}/${inference_tag}/run.sh'. You can resume the process from stage 13 using this script"
-        mkdir -p "${enh_asr_exp}/${inference_tag}"; echo "${run_args} --stage 13 \"\$@\"; exit \$?" > "${enh_asr_exp}/${inference_tag}/run.sh"; chmod +x "${enh_asr_exp}/${inference_tag}/run.sh"
+        log "Generate '${enh_asr_exp}/run_enhance.sh'. You can resume the process from stage 13 using this script"
+        mkdir -p "${enh_asr_exp}"; echo "${run_args} --stage 13 \"\$@\"; exit \$?" > "${enh_asr_exp}/run_enhance.sh"; chmod +x "${enh_asr_exp}/run_enhance.sh"
 
         for dset in ${test_sets}; do
             _data="${data_feats}/${dset}"
-            _dir="${enh_asr_exp}/${inference_tag}/${dset}"
+            _dir="${enh_asr_exp}/${inference_enh_tag}_${dset}"
             _logdir="${_dir}/logdir"
             mkdir -p "${_logdir}"
 
@@ -1330,6 +1340,7 @@ if ! "${skip_eval}"; then
                     --data_path_and_name_and_type "${_data}/${_scp},speech_mix,${_type}" \
                     --key_file "${_logdir}"/keys.JOB.scp \
                     --train_config "${enh_asr_exp}"/config.yaml \
+                    ${inference_enh_config:+--inference_config "$inference_enh_config"} \
                     --model_file "${enh_asr_exp}"/"${inference_enh_asr_model}" \
                     --output_dir "${_logdir}"/output.JOB \
                     ${_opts} ${enh_inference_args}
@@ -1472,17 +1483,17 @@ if ! "${skip_eval}"; then
         # for score_obs in true false; do
         for score_obs in true false; do
             # Peform only at the first time for observation
-            if "${score_obs}" && [ -e "${data_feats}/RESULTS.md" ]; then
-                log "${data_feats}/RESULTS.md already exists. The scoring for observation will be skipped"
+            if "${score_obs}" && [ -e "${data_feats}/RESULTS_enh.md" ]; then
+                log "${data_feats}/RESULTS_enh.md already exists. The scoring for observation will be skipped"
                 continue
             fi
 
             for dset in ${test_sets}; do
                 _data="${data_feats}/${dset}"
                 if "${score_obs}"; then
-                    _dir="${data_feats}/${dset}/scoring_enh"
+                    _dir="${data_feats}/${dset}/scoring"
                 else
-                    _dir="${enh_asr_exp}/${inference_tag}/${dset}/scoring_enh"
+                    _dir="${enh_asr_exp}/${inference_enh_tag}_${dset}/scoring"
                 fi
 
                 _logdir="${_dir}/logdir"
@@ -1508,7 +1519,7 @@ if ! "${skip_eval}"; then
                         # To compute the score of observation, input original wav.scp
                         _inf_scp+="--inf_scp ${data_feats}/${dset}/wav.scp "
                     else
-                        _inf_scp+="--inf_scp ${enh_asr_exp}/${inference_tag}/${dset}/spk${spk}.scp "
+                        _inf_scp+="--inf_scp ${enh_asr_exp}/${inference_enh_tag}_${dset}/spk${spk}.scp "
                     fi
                 done
 
@@ -1544,7 +1555,7 @@ if ! "${skip_eval}"; then
             ./scripts/utils/show_enh_score.sh "${_dir}/../.." > "${_dir}/../../RESULTS_enh.md"
         done
         log "Evaluation result for observation: ${data_feats}/RESULTS_enh.md"
-        log "Evaluation result for enhancement: ${enh_asr_exp}/enhanced/RESULTS_enh.md"
+        log "Evaluation result for enhancement: ${enh_asr_exp}/RESULTS_enh.md"
 
     fi
 else
@@ -1620,7 +1631,7 @@ if ! "${skip_upload_hf}"; then
         # shellcheck disable=SC2034     
         espnet_task=EnhS2T
         # shellcheck disable=SC2034
-        task_exp=${enh_st_exp}
+        task_exp=${enh_asr_exp}
         eval "echo \"$(cat scripts/utils/TEMPLATE_HF_Readme.md)\"" > "${dir_repo}"/README.md
 
         this_folder=${PWD}
diff --git a/egs2/TEMPLATE/enh_st1/enh_st.sh b/egs2/TEMPLATE/enh_st1/enh_st.sh
index 68eec2a7665..b6fd09c67cb 100755
--- a/egs2/TEMPLATE/enh_st1/enh_st.sh
+++ b/egs2/TEMPLATE/enh_st1/enh_st.sh
@@ -134,6 +134,8 @@ dereverb_ref_num=1
 enh_inference_args="--normalize_output_wav true"
 scoring_protocol="STOI SDR SAR SIR SI_SNR"
 ref_channel=0
+inference_enh_tag=      # Prefix to the result dir for ENH inference.
+inference_enh_config=   # Config for enhancement.
 
 # Enh Training data related
 use_dereverb_ref=false
@@ -152,8 +154,6 @@ nlsyms_txt=none  # Non-linguistic symbol list if existing.
 cleaner=none     # Text cleaner.
 g2p=none         # g2p method (needed if token_type=phn).
 lang=noinfo      # The language type of corpus.
-score_opts=                # The options given to sclite scoring
-local_score_opts=          # The options given to local/score.sh.
 enh_st_speech_fold_length=800 # fold_length for speech data during ST training.
 enh_st_text_fold_length=150   # fold_length for text data during ST training.
 lm_fold_length=150         # fold_length for LM training.
@@ -513,6 +513,14 @@ if [ -z "${inference_tag}" ]; then
     fi
 fi
 
+if [ -z "${inference_enh_tag}" ]; then
+    if [ -n "${inference_enh_config}" ]; then
+        inference_enh_tag="$(basename "${inference_enh_config}" .yaml)"
+    else
+        inference_enh_tag=enhanced
+    fi
+fi
+
 # ========================== Main stages start from here. ==========================
 
 if ! "${skip_data_prep}"; then
@@ -579,7 +587,7 @@ if ! "${skip_data_prep}"; then
                 expand_utt_extra_files=""
                 for extra_file in ${utt_extra_files}; do
                     # with regex to suuport multi-references
-                    for single_file in $(ls data/"${dset}"/${extra_file}*); do
+                    for single_file in "data/${dset}/${extra_file}"*; do
                         cp ${single_file} "${data_feats}${_suf}/${dset}"
                         expand_utt_extra_files="${expand_utt_extra_files} $(basename ${single_file})"
                     done 
@@ -731,7 +739,7 @@ if ! "${skip_data_prep}"; then
             log "Merge src and target data if joint BPE"
 
             cat $tgt_bpe_train_text > ${data_feats}/${train_set}/text.${src_lang}_${tgt_lang}
-            [ ! -z "${src_bpe_train_text}" ] && cat ${src_bpe_train_text} >> ${data_feats}/${train_set}/text.${src_lang}_${tgt_lang}
+            [ -n "${src_bpe_train_text}" ] && cat ${src_bpe_train_text} >> ${data_feats}/${train_set}/text.${src_lang}_${tgt_lang}
             # Set the new text as the target text
             tgt_bpe_train_text="${data_feats}/${train_set}/text.${src_lang}_${tgt_lang}"
         fi
@@ -1435,7 +1443,7 @@ if ! "${skip_eval}"; then
                     --st_train_config "${enh_st_exp}"/config.yaml \
                     --st_model_file "${enh_st_exp}"/"${inference_enh_st_model}" \
                     --output_dir "${_logdir}"/output.JOB \
-                    ${_opts} ${inference_args}
+                    ${_opts} ${st_inference_args}
 
             # 3. Concatenates the output files from each jobs
             for f in token token_int score text; do
@@ -1460,12 +1468,12 @@ if ! "${skip_eval}"; then
         _opts=
 
         # 2. Generate run.sh
-        log "Generate '${enh_st_exp}/${inference_tag}/run.sh'. You can resume the process from stage 13 using this script"
-        mkdir -p "${enh_st_exp}/${inference_tag}"; echo "${run_args} --stage 13 \"\$@\"; exit \$?" > "${enh_st_exp}/${inference_tag}/run.sh"; chmod +x "${enh_st_exp}/${inference_tag}/run.sh"
+        log "Generate '${enh_st_exp}/run_enhance.sh'. You can resume the process from stage 13 using this script"
+        mkdir -p "${enh_st_exp}"; echo "${run_args} --stage 13 \"\$@\"; exit \$?" > "${enh_st_exp}/run_enhance.sh"; chmod +x "${enh_st_exp}/run_enhance.sh"
 
         for dset in ${test_sets}; do
             _data="${data_feats}/${dset}"
-            _dir="${enh_st_exp}/${inference_tag}/${dset}"
+            _dir="${enh_st_exp}/${inference_enh_tag}_${dset}"
             _logdir="${_dir}/logdir"
             mkdir -p "${_logdir}"
 
@@ -1497,6 +1505,7 @@ if ! "${skip_eval}"; then
                     --data_path_and_name_and_type "${_data}/${_scp},speech_mix,${_type}" \
                     --key_file "${_logdir}"/keys.JOB.scp \
                     --train_config "${enh_st_exp}"/config.yaml \
+                    ${inference_enh_config:+--inference_config "$inference_enh_config"} \
                     --model_file "${enh_st_exp}"/"${inference_enh_st_model}" \
                     --output_dir "${_logdir}"/output.JOB \
                     ${_opts} ${enh_inference_args}
@@ -1521,7 +1530,7 @@ if ! "${skip_eval}"; then
 
         for dset in ${test_sets}; do
             _data="${data_feats}/${dset}"
-            _dir="${st_exp}/${inference_tag}/${dset}"
+            _dir="${enh_st_exp}/${inference_tag}/${dset}"
 
             # TODO(jiatong): add asr scoring and inference
 
@@ -1637,17 +1646,17 @@ if ! "${skip_eval}"; then
         # for score_obs in true false; do
         for score_obs in true false; do
             # Peform only at the first time for observation
-            if "${score_obs}" && [ -e "${data_feats}/RESULTS.md" ]; then
-                log "${data_feats}/RESULTS.md already exists. The scoring for observation will be skipped"
+            if "${score_obs}" && [ -e "${data_feats}/RESULTS_enh.md" ]; then
+                log "${data_feats}/RESULTS_enh.md already exists. The scoring for observation will be skipped"
                 continue
             fi
 
             for dset in ${test_sets}; do
                 _data="${data_feats}/${dset}"
                 if "${score_obs}"; then
-                    _dir="${data_feats}/${dset}/scoring_enh"
+                    _dir="${data_feats}/${dset}/scoring"
                 else
-                    _dir="${enh_st_exp}/${inference_tag}/${dset}/scoring_enh"
+                    _dir="${enh_st_exp}/${inference_enh_tag}_${dset}/scoring"
                 fi
 
                 _logdir="${_dir}/logdir"
@@ -1673,7 +1682,7 @@ if ! "${skip_eval}"; then
                         # To compute the score of observation, input original wav.scp
                         _inf_scp+="--inf_scp ${data_feats}/${dset}/wav.scp "
                     else
-                        _inf_scp+="--inf_scp ${enh_st_exp}/${inference_tag}/${dset}/spk${spk}.scp "
+                        _inf_scp+="--inf_scp ${enh_st_exp}/${inference_enh_tag}_${dset}/spk${spk}.scp "
                     fi
                 done
 
@@ -1709,7 +1718,7 @@ if ! "${skip_eval}"; then
             ./scripts/utils/show_enh_score.sh "${_dir}/../.." > "${_dir}/../../RESULTS_enh.md"
         done
         log "Evaluation result for observation: ${data_feats}/RESULTS_enh.md"
-        log "Evaluation result for enhancement: ${enh_asr_exp}/enhanced/RESULTS_enh.md"
+        log "Evaluation result for enhancement: ${enh_st_exp}/RESULTS_enh.md"
 
     fi
 else

From e7b0d6b5ff27743c9e79f1ea2a14efddc6f37e28 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Wed, 20 Apr 2022 21:55:29 +0800
Subject: [PATCH 31/56] Fix typos in comments

---
 espnet2/enh/layers/beamformer.py           | 4 ++--
 espnet2/enh/loss/criterions/time_domain.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/espnet2/enh/layers/beamformer.py b/espnet2/enh/layers/beamformer.py
index 1e72b99071f..e3d61d2489f 100644
--- a/espnet2/enh/layers/beamformer.py
+++ b/espnet2/enh/layers/beamformer.py
@@ -443,7 +443,7 @@ def get_sdw_mwf_vector(
         filtering for noise reduction; A. Spriet et al, 2004
         https://dl.acm.org/doi/abs/10.1016/j.sigpro.2004.07.028
         [2] Rank-1 constrained multichannel Wiener filter for speech recognition in
-        noisy environments; Z. Wangyou et al, 2018
+        noisy environments; Z. Wang et al, 2018
         https://hal.inria.fr/hal-01634449/document
         [3] Low-rank approximation based multichannel Wiener filter algorithms for
         noise reduction with application in cochlear implants; R. Serizel, 2014
@@ -525,7 +525,7 @@ def get_rank1_mwf_vector(
 
     Reference:
         [1] Rank-1 constrained multichannel Wiener filter for speech recognition in
-        noisy environments; Z. Wangyou et al, 2018
+        noisy environments; Z. Wang et al, 2018
         https://hal.inria.fr/hal-01634449/document
         [2] Low-rank approximation based multichannel Wiener filter algorithms for
         noise reduction with application in cochlear implants; R. Serizel, 2014
diff --git a/espnet2/enh/loss/criterions/time_domain.py b/espnet2/enh/loss/criterions/time_domain.py
index 704672fcd5d..e2060079c70 100644
--- a/espnet2/enh/loss/criterions/time_domain.py
+++ b/espnet2/enh/loss/criterions/time_domain.py
@@ -10,7 +10,7 @@
 
 
 class TimeDomainLoss(AbsEnhLoss, ABC):
-    """Base class for all frequence-domain Enhancement loss modules."""
+    """Base class for all time-domain Enhancement loss modules."""
 
     pass
 

From 0d20fa7ab6837f5003a626130add6a211a5c0a9b Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Thu, 21 Apr 2022 19:46:51 +0800
Subject: [PATCH 32/56] Fix a bug in utterance_mvn

This PR fix the bug mentioned in https://github.com/espnet/espnet/issues/4296.
---
 espnet2/layers/utterance_mvn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/espnet2/layers/utterance_mvn.py b/espnet2/layers/utterance_mvn.py
index a41f869f322..4f1adb3e53b 100644
--- a/espnet2/layers/utterance_mvn.py
+++ b/espnet2/layers/utterance_mvn.py
@@ -76,7 +76,7 @@ def utterance_mvn(
         if norm_vars:
             var = x.pow(2).sum(dim=1, keepdim=True) / ilens_
             std = torch.clamp(var.sqrt(), min=eps)
-            x = x / std.sqrt()
+            x = x / std
         return x, ilens
     else:
         if norm_vars:

From d280e649fa4f726e209d0aab8f5bc349c7b9c710 Mon Sep 17 00:00:00 2001
From: Jiatong <728307998@qq.com>
Date: Fri, 22 Apr 2022 11:44:23 -0400
Subject: [PATCH 33/56] Update egs2/covost2/asr1/run.sh

Co-authored-by: Chaitanya Narisetty <cnariset@andrew.cmu.edu>
---
 egs2/covost2/asr1/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/egs2/covost2/asr1/run.sh b/egs2/covost2/asr1/run.sh
index 4bead69a372..e40acf30dd7 100755
--- a/egs2/covost2/asr1/run.sh
+++ b/egs2/covost2/asr1/run.sh
@@ -89,7 +89,7 @@ fi
     --token_type bpe \
     --nbpe $nbpe \
     --feats_type raw \
-    --speed_perturb_factors ${speed_perturb_factors} \
+    --speed_perturb_factors "${speed_perturb_factors}" \
     --asr_config "${asr_config}" \
     --inference_config "${inference_config}" \
     --train_set "${train_set}" \

From 488d106e3c5ffb94cbc280cbbd3f9545e1e033ff Mon Sep 17 00:00:00 2001
From: Jiatong <728307998@qq.com>
Date: Fri, 22 Apr 2022 11:44:30 -0400
Subject: [PATCH 34/56] Update egs2/covost2/st1/run.sh

Co-authored-by: Chaitanya Narisetty <cnariset@andrew.cmu.edu>
---
 egs2/covost2/st1/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/egs2/covost2/st1/run.sh b/egs2/covost2/st1/run.sh
index b24fd81030c..dda6432f5e0 100755
--- a/egs2/covost2/st1/run.sh
+++ b/egs2/covost2/st1/run.sh
@@ -91,7 +91,7 @@ fi
 ./st.sh \
     --ngpu 1 \
     --local_data_opts "--stage 0 --src_lang ${src_lang} --tgt_lang ${tgt_lang}" \
-    --speed_perturb_factors ${speed_perturb_factors} \
+    --speed_perturb_factors "${speed_perturb_factors}" \
     --use_lm false \
     --feats_type raw \
     --audio_format "flac.ark" \

From 55644cbc57495d7f891d5c055f3ac63e02f303a6 Mon Sep 17 00:00:00 2001
From: Jiatong <728307998@qq.com>
Date: Fri, 22 Apr 2022 11:44:36 -0400
Subject: [PATCH 35/56] Update egs2/covost2/st1/run.sh

Co-authored-by: Chaitanya Narisetty <cnariset@andrew.cmu.edu>
---
 egs2/covost2/st1/run.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/egs2/covost2/st1/run.sh b/egs2/covost2/st1/run.sh
index dda6432f5e0..5cd66dbaf53 100755
--- a/egs2/covost2/st1/run.sh
+++ b/egs2/covost2/st1/run.sh
@@ -95,7 +95,6 @@ fi
     --use_lm false \
     --feats_type raw \
     --audio_format "flac.ark" \
-    --use_lm false \
     --token_joint false \
     --src_lang ${src_lang} \
     --tgt_lang ${tgt_lang} \

From 0eaf24321b2dc4de66cad4a3d278cb01bf51ad80 Mon Sep 17 00:00:00 2001
From: Jiatong <728307998@qq.com>
Date: Fri, 22 Apr 2022 11:44:43 -0400
Subject: [PATCH 36/56] Update egs2/covost2/mt1/run.sh

Co-authored-by: Chaitanya Narisetty <cnariset@andrew.cmu.edu>
---
 egs2/covost2/mt1/run.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/egs2/covost2/mt1/run.sh b/egs2/covost2/mt1/run.sh
index 57e22970d94..11265d7ec87 100755
--- a/egs2/covost2/mt1/run.sh
+++ b/egs2/covost2/mt1/run.sh
@@ -80,7 +80,6 @@ fi
     --use_lm false \
     --feats_type raw \
     --audio_format "flac.ark" \
-    --use_lm false \
     --token_joint false \
     --src_lang ${src_lang} \
     --tgt_lang ${tgt_lang} \

From 1ab456c342c573ce93b71700a5507fdb71989350 Mon Sep 17 00:00:00 2001
From: Ashley Wu <wangchew@andrew.cmu.edu>
Date: Sat, 23 Apr 2022 14:01:17 -0400
Subject: [PATCH 37/56] clean up

---
 .../asr1/local/download_and_untar.sh            |  2 +-
 egs2/mediaspeech/asr1/local/data.sh             | 17 ++++++++---------
 egs2/mediaspeech/asr1/local/data_prep.pl        |  1 -
 3 files changed, 9 insertions(+), 11 deletions(-)
 delete mode 120000 egs2/mediaspeech/asr1/local/data_prep.pl

diff --git a/egs/commonvoice/asr1/local/download_and_untar.sh b/egs/commonvoice/asr1/local/download_and_untar.sh
index 1f5c40d9b0e..cce26302127 100755
--- a/egs/commonvoice/asr1/local/download_and_untar.sh
+++ b/egs/commonvoice/asr1/local/download_and_untar.sh
@@ -16,7 +16,7 @@ fi
 
 if [ $# -ne 3 ]; then
   echo "Usage: $0 [--remove-archive] <data-base> <url> <filename>"
-  echo "e.g.: $0 /export/data/ https://common-voice-data-download.s3.amazonaws.com/cv_corpus_v1.tar.gz cv_corpus_v1.tar.gz"
+  echo "e.g.: $0 /export/data/ https://us.openslr.org/resources/108/FR.tgz"
   echo "With --remove-archive it will remove the archive after successfully un-tarring it."
   exit 0;
 fi
diff --git a/egs2/mediaspeech/asr1/local/data.sh b/egs2/mediaspeech/asr1/local/data.sh
index 9a6d54d950e..a904d143883 100755
--- a/egs2/mediaspeech/asr1/local/data.sh
+++ b/egs2/mediaspeech/asr1/local/data.sh
@@ -19,9 +19,9 @@ log() {
     echo -e "$(date '+%Y-%m-%dT%H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
 }
 
-mkdir -p ${COMMONVOICE}
-if [ -z "${COMMONVOICE}" ]; then
-    log "Fill the value of 'COMMONVOICE' of db.sh"
+mkdir -p ${MEDIASPEECH}
+if [ -z "${MEDIASPEECH}" ]; then
+    log "Fill the value of 'MEDIASPEECH' of db.sh"
     exit 1
 fi
 
@@ -34,18 +34,17 @@ set -o pipefail
 log "data preparation started"
 
 if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then 
-    log "stage1: Download data to ${COMMONVOICE}"
-    log "The default data of this recipe is from commonvoice 5.1, for newer version, you need to register at \
-         https://commonvoice.mozilla.org/"
-    local/download_and_untar.sh ${COMMONVOICE} ${data_url} FR.tgz
+    log "stage1: Download data to ${MEDIASPEECH}"
+    log "The default data of this recipe is from mediaspeech - french 
+    local/download_and_untar.sh ${MEDIASPEECH} ${data_url} FR.tgz
 fi
 
 if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
-    log "stage2: Preparing data for commonvoice"
+    log "stage2: Preparing data for mediaspeech"
     ### Task dependent. You have to make data the following preparation part by yourself.
     mkdir -p data/{train_as,dev_as,test_as,validated_as}
     python3 local/data_prep.py \
-        --data_path ${COMMONVOICE}/FR \
+        --data_path ${MEDIASPEECH}/FR \
         --train_dir data/train_as \
         --dev_dir data/dev_as \
         --test_dir data/test_as \
diff --git a/egs2/mediaspeech/asr1/local/data_prep.pl b/egs2/mediaspeech/asr1/local/data_prep.pl
deleted file mode 120000
index 28fe16ef888..00000000000
--- a/egs2/mediaspeech/asr1/local/data_prep.pl
+++ /dev/null
@@ -1 +0,0 @@
-../../../../egs/commonvoice/asr1/local/data_prep.pl
\ No newline at end of file

From a8e5d3d0ef346728a1be74166d4030370b7386cf Mon Sep 17 00:00:00 2001
From: Yoshiki <mas03151997@gmail.com>
Date: Sun, 24 Apr 2022 04:35:33 +0900
Subject: [PATCH 38/56] modify config and add README

---
 egs2/dns_icassp21/enh1/README.md              | 22 +++++++++++++++++++
 .../enh1/conf/tuning/train_enh_tcn_tf.yaml    |  4 ++--
 2 files changed, 24 insertions(+), 2 deletions(-)
 create mode 100644 egs2/dns_icassp21/enh1/README.md

diff --git a/egs2/dns_icassp21/enh1/README.md b/egs2/dns_icassp21/enh1/README.md
new file mode 100644
index 00000000000..7054ca71f27
--- /dev/null
+++ b/egs2/dns_icassp21/enh1/README.md
@@ -0,0 +1,22 @@
+<!-- Generated by ./scripts/utils/show_enh_score.sh -->
+# RESULTS
+## Environments
+- date: `Thu Apr 21 21:49:46 UTC 2022`
+- python version: `3.7.4 (default, Aug 13 2019, 20:35:49)  [GCC 7.3.0]`
+- espnet version: `espnet 202204`
+- pytorch version: `pytorch 1.10.1+cu111`
+- Git hash: ``
+  - Commit date: ``
+
+
+## enh_train_enh_tcn_tf_raw
+
+- config: ./conf/tuning/train_enh_tcn_tf.yaml
+- Pretrained model: https://huggingface.co/espnet/dns_icassp21_enh_train_enh_tcn_tf_raw
+
+
+|dataset|STOI|SAR|SDR|SIR|SI_SNR|
+|---|---|---|---|---|---|
+|enhanced_cv_synthetic|0.93|18.96|18.96|0.00|18.79|
+|enhanced_tt_synthetic_track_1|0.77|14.19|14.19|0.00|12.15|
+
diff --git a/egs2/dns_icassp21/enh1/conf/tuning/train_enh_tcn_tf.yaml b/egs2/dns_icassp21/enh1/conf/tuning/train_enh_tcn_tf.yaml
index f3c433a4f44..aedfd53b710 100644
--- a/egs2/dns_icassp21/enh1/conf/tuning/train_enh_tcn_tf.yaml
+++ b/egs2/dns_icassp21/enh1/conf/tuning/train_enh_tcn_tf.yaml
@@ -2,7 +2,7 @@ optim: adam
 init: xavier_uniform
 max_epoch: 100
 batch_type: folded
-batch_size: 4
+batch_size: 128
 num_workers: 4
 optim_conf:
     lr: 1.0e-03
@@ -37,7 +37,7 @@ decoder_conf:
 separator: tcn
 separator_conf:
     num_spk: 1
-    layer: 8
+    layer: 4
     stack: 3
     bottleneck_dim: 128
     hidden_dim: 512

From d3151222797882db6f42371d356ac55fe8712d07 Mon Sep 17 00:00:00 2001
From: Yoshiki <mas03151997@gmail.com>
Date: Sun, 24 Apr 2022 04:45:20 +0900
Subject: [PATCH 39/56] Update README

---
 egs2/dns_icassp21/enh1/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/egs2/dns_icassp21/enh1/README.md b/egs2/dns_icassp21/enh1/README.md
index 7054ca71f27..4f00b25052b 100644
--- a/egs2/dns_icassp21/enh1/README.md
+++ b/egs2/dns_icassp21/enh1/README.md
@@ -5,8 +5,8 @@
 - python version: `3.7.4 (default, Aug 13 2019, 20:35:49)  [GCC 7.3.0]`
 - espnet version: `espnet 202204`
 - pytorch version: `pytorch 1.10.1+cu111`
-- Git hash: ``
-  - Commit date: ``
+- Git hash: `a8e5d3d0ef346728a1be74166d4030370b7386cf`
+  - Commit date: `Sun Apr 24 04:35:33 2022 +0900`
 
 
 ## enh_train_enh_tcn_tf_raw

From 712dee77a63a47e4efd4043c772095a73739915a Mon Sep 17 00:00:00 2001
From: Yoshiki Masuyama <40336669+YoshikiMas@users.noreply.github.com>
Date: Sat, 23 Apr 2022 22:45:54 -0400
Subject: [PATCH 40/56] Update egs2/dns_icassp21/enh1/README.md

Co-authored-by: Shinji Watanabe <sw005320@gmail.com>
---
 egs2/dns_icassp21/enh1/README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/egs2/dns_icassp21/enh1/README.md b/egs2/dns_icassp21/enh1/README.md
index 4f00b25052b..c5f182df1ab 100644
--- a/egs2/dns_icassp21/enh1/README.md
+++ b/egs2/dns_icassp21/enh1/README.md
@@ -15,8 +15,8 @@
 - Pretrained model: https://huggingface.co/espnet/dns_icassp21_enh_train_enh_tcn_tf_raw
 
 
-|dataset|STOI|SAR|SDR|SIR|SI_SNR|
-|---|---|---|---|---|---|
-|enhanced_cv_synthetic|0.93|18.96|18.96|0.00|18.79|
-|enhanced_tt_synthetic_track_1|0.77|14.19|14.19|0.00|12.15|
+|dataset|STOI|SAR|SDR|SI_SNR|
+|---|---|---|---|---|
+|enhanced_cv_synthetic|0.93|18.96|18.96|18.79|
+|enhanced_tt_synthetic_track_1|0.77|14.19|14.19|12.15|
 

From 4b0e934480067da5f5a117e072721f69eb06157b Mon Sep 17 00:00:00 2001
From: Yoshiki <mas03151997@gmail.com>
Date: Sun, 24 Apr 2022 15:58:22 +0900
Subject: [PATCH 41/56] update README

---
 egs2/dns_icassp21/enh1/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/egs2/dns_icassp21/enh1/README.md b/egs2/dns_icassp21/enh1/README.md
index 4f00b25052b..d36fef31c8a 100644
--- a/egs2/dns_icassp21/enh1/README.md
+++ b/egs2/dns_icassp21/enh1/README.md
@@ -15,8 +15,8 @@
 - Pretrained model: https://huggingface.co/espnet/dns_icassp21_enh_train_enh_tcn_tf_raw
 
 
-|dataset|STOI|SAR|SDR|SIR|SI_SNR|
+|dataset|STOI|SDR|SI_SNR|
 |---|---|---|---|---|---|
-|enhanced_cv_synthetic|0.93|18.96|18.96|0.00|18.79|
-|enhanced_tt_synthetic_track_1|0.77|14.19|14.19|0.00|12.15|
+|enhanced_cv_synthetic|0.93|18.96|18.79|
+|enhanced_tt_synthetic_track_1|0.77|14.19|12.15|
 

From ca481bc698d9690a364b3cc83220d7de8ce6188f Mon Sep 17 00:00:00 2001
From: Yoshiki <mas03151997@gmail.com>
Date: Sun, 24 Apr 2022 16:06:01 +0900
Subject: [PATCH 42/56] fix README.md

---
 egs2/dns_icassp21/enh1/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/egs2/dns_icassp21/enh1/README.md b/egs2/dns_icassp21/enh1/README.md
index d36fef31c8a..c10aa6ed255 100644
--- a/egs2/dns_icassp21/enh1/README.md
+++ b/egs2/dns_icassp21/enh1/README.md
@@ -16,7 +16,7 @@
 
 
 |dataset|STOI|SDR|SI_SNR|
-|---|---|---|---|---|---|
+|---|---|---|---|
 |enhanced_cv_synthetic|0.93|18.96|18.79|
 |enhanced_tt_synthetic_track_1|0.77|14.19|12.15|
 

From 1266a292deafa9082c1f942d4288e9780bace775 Mon Sep 17 00:00:00 2001
From: simpleoier <netnetchangxk@gmail.com>
Date: Sun, 24 Apr 2022 11:11:46 -0400
Subject: [PATCH 43/56] add integration test and fix the decoding in enh_asr
 and enh_st

---
 ci/test_integration_espnet2.sh                |  10 ++
 egs2/TEMPLATE/enh_asr1/enh_asr.sh             |   3 +-
 egs2/TEMPLATE/enh_st1/enh_st.sh               |   1 +
 egs2/mini_an4/enh_asr1/cmd.sh                 | 110 ++++++++++++++++++
 egs2/mini_an4/enh_asr1/conf/fbank.conf        |   2 +
 egs2/mini_an4/enh_asr1/conf/pbs.conf          |  11 ++
 egs2/mini_an4/enh_asr1/conf/pitch.conf        |   1 +
 egs2/mini_an4/enh_asr1/conf/queue.conf        |  12 ++
 egs2/mini_an4/enh_asr1/conf/slurm.conf        |  14 +++
 egs2/mini_an4/enh_asr1/db.sh                  |   1 +
 egs2/mini_an4/enh_asr1/downloads.tar.gz       |   1 +
 egs2/mini_an4/enh_asr1/enh_asr.sh             |   1 +
 egs2/mini_an4/enh_asr1/local/data.sh          |  94 +++++++++++++++
 egs2/mini_an4/enh_asr1/local/data_prep.py     |   1 +
 .../enh_asr1/local/download_and_untar.sh      |   1 +
 egs2/mini_an4/enh_asr1/local/path.sh          |   0
 egs2/mini_an4/enh_asr1/path.sh                |   1 +
 egs2/mini_an4/enh_asr1/pyscripts              |   1 +
 egs2/mini_an4/enh_asr1/run.sh                 |  13 +++
 egs2/mini_an4/enh_asr1/scripts                |   1 +
 egs2/mini_an4/enh_asr1/steps                  |   1 +
 egs2/mini_an4/enh_asr1/utils                  |   1 +
 espnet2/enh/espnet_enh_s2t_model.py           |   3 +
 test/espnet2/bin/test_asr_inference.py        |   2 +-
 24 files changed, 284 insertions(+), 2 deletions(-)
 create mode 100644 egs2/mini_an4/enh_asr1/cmd.sh
 create mode 100644 egs2/mini_an4/enh_asr1/conf/fbank.conf
 create mode 100644 egs2/mini_an4/enh_asr1/conf/pbs.conf
 create mode 100644 egs2/mini_an4/enh_asr1/conf/pitch.conf
 create mode 100644 egs2/mini_an4/enh_asr1/conf/queue.conf
 create mode 100644 egs2/mini_an4/enh_asr1/conf/slurm.conf
 create mode 120000 egs2/mini_an4/enh_asr1/db.sh
 create mode 120000 egs2/mini_an4/enh_asr1/downloads.tar.gz
 create mode 120000 egs2/mini_an4/enh_asr1/enh_asr.sh
 create mode 100755 egs2/mini_an4/enh_asr1/local/data.sh
 create mode 120000 egs2/mini_an4/enh_asr1/local/data_prep.py
 create mode 120000 egs2/mini_an4/enh_asr1/local/download_and_untar.sh
 create mode 100644 egs2/mini_an4/enh_asr1/local/path.sh
 create mode 120000 egs2/mini_an4/enh_asr1/path.sh
 create mode 120000 egs2/mini_an4/enh_asr1/pyscripts
 create mode 100755 egs2/mini_an4/enh_asr1/run.sh
 create mode 120000 egs2/mini_an4/enh_asr1/scripts
 create mode 120000 egs2/mini_an4/enh_asr1/steps
 create mode 120000 egs2/mini_an4/enh_asr1/utils

diff --git a/ci/test_integration_espnet2.sh b/ci/test_integration_espnet2.sh
index 78086272af7..3cf663466e7 100755
--- a/ci/test_integration_espnet2.sh
+++ b/ci/test_integration_espnet2.sh
@@ -100,6 +100,16 @@ if python3 -c "import fairseq" &> /dev/null; then
     cd "${cwd}"
 fi
 
+# [ESPnet2] test enh_asr1 recipe
+if python -c 'import torch as t; from distutils.version import LooseVersion as L; assert L(t.__version__) >= L("1.2.0")' &> /dev/null;  then
+    cd ./egs2/mini_an4/enh_asr1
+    echo "==== [ESPnet2] ENH_ASR ==="
+    ./run.sh --ngpu 0 --stage 0 --stop-stage 15 --skip-upload_hf false --feats-type "raw" --spk-num 1 --enh_asr_args "--max_epoch=1 --enh_separator_conf num_spk=1" --python "${python}"
+    # Remove generated files in order to reduce the disk usage
+    rm -rf exp dump data
+    cd "${cwd}"
+fi
+
 # [ESPnet2] Validate configuration files
 echo "<blank>" > dummy_token_list
 echo "==== [ESPnet2] Validation configuration files ==="
diff --git a/egs2/TEMPLATE/enh_asr1/enh_asr.sh b/egs2/TEMPLATE/enh_asr1/enh_asr.sh
index 3d238938a57..70c2e791046 100755
--- a/egs2/TEMPLATE/enh_asr1/enh_asr.sh
+++ b/egs2/TEMPLATE/enh_asr1/enh_asr.sh
@@ -553,7 +553,7 @@ if ! "${skip_data_prep}"; then
                     # shellcheck disable=SC2086
                     scripts/audio/format_wav_scp.sh --nj "${nj}" --cmd "${train_cmd}" \
                         --out-filename "${spk}.scp" \
-                        --ref_channels "0" \
+                        --ref_channels "${ref_channel}" \
                         --audio-format "${audio_format}" --fs "${fs}" ${_opts} \
                         "data/${dset}/${spk}.scp" "${data_feats}${_suf}/${dset}" \
                         "${data_feats}${_suf}/${dset}/logs/${spk}" "${data_feats}${_suf}/${dset}/data/${spk}"
@@ -1259,6 +1259,7 @@ if ! "${skip_eval}"; then
             # shellcheck disable=SC2086
             ${_cmd} --gpu "${_ngpu}" JOB=1:"${_nj}" "${_logdir}"/asr_inference.JOB.log \
                 ${python} -m ${asr_inference_tool} \
+                    --enh_s2t_task true \
                     --batch_size ${batch_size} \
                     --ngpu "${_ngpu}" \
                     --data_path_and_name_and_type "${_data}/${_scp},speech,${_type}" \
diff --git a/egs2/TEMPLATE/enh_st1/enh_st.sh b/egs2/TEMPLATE/enh_st1/enh_st.sh
index 68eec2a7665..3c08bf704cc 100755
--- a/egs2/TEMPLATE/enh_st1/enh_st.sh
+++ b/egs2/TEMPLATE/enh_st1/enh_st.sh
@@ -1428,6 +1428,7 @@ if ! "${skip_eval}"; then
             # shellcheck disable=SC2086
             ${_cmd} --gpu "${_ngpu}" JOB=1:"${_nj}" "${_logdir}"/st_inference.JOB.log \
                 ${python} -m ${st_inference_tool} \
+                    --enh_s2t_task true \
                     --batch_size ${batch_size} \
                     --ngpu "${_ngpu}" \
                     --data_path_and_name_and_type "${_data}/${_scp},speech,${_type}" \
diff --git a/egs2/mini_an4/enh_asr1/cmd.sh b/egs2/mini_an4/enh_asr1/cmd.sh
new file mode 100644
index 00000000000..2aae6919fef
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/cmd.sh
@@ -0,0 +1,110 @@
+# ====== About run.pl, queue.pl, slurm.pl, and ssh.pl ======
+# Usage: <cmd>.pl [options] JOB=1:<nj> <log> <command...>
+# e.g.
+#   run.pl --mem 4G JOB=1:10 echo.JOB.log echo JOB
+#
+# Options:
+#   --time <time>: Limit the maximum time to execute.
+#   --mem <mem>: Limit the maximum memory usage.
+#   -–max-jobs-run <njob>: Limit the number parallel jobs. This is ignored for non-array jobs.
+#   --num-threads <ngpu>: Specify the number of CPU core.
+#   --gpu <ngpu>: Specify the number of GPU devices.
+#   --config: Change the configuration file from default.
+#
+# "JOB=1:10" is used for "array jobs" and it can control the number of parallel jobs.
+# The left string of "=", i.e. "JOB", is replaced by <N>(Nth job) in the command and the log file name,
+# e.g. "echo JOB" is changed to "echo 3" for the 3rd job and "echo 8" for 8th job respectively.
+# Note that the number must start with a positive number, so you can't use "JOB=0:10" for example.
+#
+# run.pl, queue.pl, slurm.pl, and ssh.pl have unified interface, not depending on its backend.
+# These options are mapping to specific options for each backend and
+# it is configured by "conf/queue.conf" and "conf/slurm.conf" by default.
+# If jobs failed, your configuration might be wrong for your environment.
+#
+#
+# The official documentation for run.pl, queue.pl, slurm.pl, and ssh.pl:
+#   "Parallelization in Kaldi": http://kaldi-asr.org/doc/queue.html
+# =========================================================~
+
+
+# Select the backend used by run.sh from "local", "stdout", "sge", "slurm", or "ssh"
+cmd_backend='local'
+
+# Local machine, without any Job scheduling system
+if [ "${cmd_backend}" = local ]; then
+
+    # The other usage
+    export train_cmd="run.pl"
+    # Used for "*_train.py": "--gpu" is appended optionally by run.sh
+    export cuda_cmd="run.pl"
+    # Used for "*_recog.py"
+    export decode_cmd="run.pl"
+
+# Local machine logging to stdout and log file, without any Job scheduling system
+elif [ "${cmd_backend}" = stdout ]; then
+
+    # The other usage
+    export train_cmd="stdout.pl"
+    # Used for "*_train.py": "--gpu" is appended optionally by run.sh
+    export cuda_cmd="stdout.pl"
+    # Used for "*_recog.py"
+    export decode_cmd="stdout.pl"
+
+
+# "qsub" (Sun Grid Engine, or derivation of it)
+elif [ "${cmd_backend}" = sge ]; then
+    # The default setting is written in conf/queue.conf.
+    # You must change "-q g.q" for the "queue" for your environment.
+    # To know the "queue" names, type "qhost -q"
+    # Note that to use "--gpu *", you have to setup "complex_value" for the system scheduler.
+
+    export train_cmd="queue.pl"
+    export cuda_cmd="queue.pl"
+    export decode_cmd="queue.pl"
+
+
+# "qsub" (Torque/PBS.)
+elif [ "${cmd_backend}" = pbs ]; then
+    # The default setting is written in conf/pbs.conf.
+
+    export train_cmd="pbs.pl"
+    export cuda_cmd="pbs.pl"
+    export decode_cmd="pbs.pl"
+
+
+# "sbatch" (Slurm)
+elif [ "${cmd_backend}" = slurm ]; then
+    # The default setting is written in conf/slurm.conf.
+    # You must change "-p cpu" and "-p gpu" for the "partition" for your environment.
+    # To know the "partion" names, type "sinfo".
+    # You can use "--gpu * " by default for slurm and it is interpreted as "--gres gpu:*"
+    # The devices are allocated exclusively using "${CUDA_VISIBLE_DEVICES}".
+
+    export train_cmd="slurm.pl"
+    export cuda_cmd="slurm.pl"
+    export decode_cmd="slurm.pl"
+
+elif [ "${cmd_backend}" = ssh ]; then
+    # You have to create ".queue/machines" to specify the host to execute jobs.
+    # e.g. .queue/machines
+    #   host1
+    #   host2
+    #   host3
+    # Assuming you can login them without any password, i.e. You have to set ssh keys.
+
+    export train_cmd="ssh.pl"
+    export cuda_cmd="ssh.pl"
+    export decode_cmd="ssh.pl"
+
+# This is an example of specifying several unique options in the JHU CLSP cluster setup.
+# Users can modify/add their own command options according to their cluster environments.
+elif [ "${cmd_backend}" = jhu ]; then
+
+    export train_cmd="queue.pl --mem 2G"
+    export cuda_cmd="queue-freegpu.pl --mem 2G --gpu 1 --config conf/queue.conf"
+    export decode_cmd="queue.pl --mem 4G"
+
+else
+    echo "$0: Error: Unknown cmd_backend=${cmd_backend}" 1>&2
+    return 1
+fi
diff --git a/egs2/mini_an4/enh_asr1/conf/fbank.conf b/egs2/mini_an4/enh_asr1/conf/fbank.conf
new file mode 100644
index 00000000000..82ac7bd0dbc
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/conf/fbank.conf
@@ -0,0 +1,2 @@
+--sample-frequency=16000 
+--num-mel-bins=80
diff --git a/egs2/mini_an4/enh_asr1/conf/pbs.conf b/egs2/mini_an4/enh_asr1/conf/pbs.conf
new file mode 100644
index 00000000000..119509938ce
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/conf/pbs.conf
@@ -0,0 +1,11 @@
+# Default configuration
+command qsub -V -v PATH -S /bin/bash
+option name=* -N $0
+option mem=* -l mem=$0
+option mem=0          # Do not add anything to qsub_opts
+option num_threads=* -l ncpus=$0
+option num_threads=1  # Do not add anything to qsub_opts
+option num_nodes=* -l nodes=$0:ppn=1
+default gpu=0
+option gpu=0
+option gpu=* -l ngpus=$0
diff --git a/egs2/mini_an4/enh_asr1/conf/pitch.conf b/egs2/mini_an4/enh_asr1/conf/pitch.conf
new file mode 100644
index 00000000000..e959a19d5b8
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/conf/pitch.conf
@@ -0,0 +1 @@
+--sample-frequency=16000
diff --git a/egs2/mini_an4/enh_asr1/conf/queue.conf b/egs2/mini_an4/enh_asr1/conf/queue.conf
new file mode 100644
index 00000000000..500582fab31
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/conf/queue.conf
@@ -0,0 +1,12 @@
+# Default configuration
+command qsub -v PATH -cwd -S /bin/bash -j y -l arch=*64*
+option name=* -N $0
+option mem=* -l mem_free=$0,ram_free=$0
+option mem=0          # Do not add anything to qsub_opts
+option num_threads=* -pe smp $0
+option num_threads=1  # Do not add anything to qsub_opts
+option max_jobs_run=* -tc $0
+option num_nodes=* -pe mpi $0  # You must set this PE as allocation_rule=1
+default gpu=0
+option gpu=0
+option gpu=* -l gpu=$0 -q g.q
diff --git a/egs2/mini_an4/enh_asr1/conf/slurm.conf b/egs2/mini_an4/enh_asr1/conf/slurm.conf
new file mode 100644
index 00000000000..3b229673638
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/conf/slurm.conf
@@ -0,0 +1,14 @@
+# Default configuration
+command sbatch --export=PATH
+option name=* --job-name $0
+option time=* --time $0
+option mem=* --mem-per-cpu $0
+option mem=0
+option num_threads=* --cpus-per-task $0
+option num_threads=1 --cpus-per-task 1
+option num_nodes=* --nodes $0
+default gpu=0
+option gpu=0 -p cpu
+option gpu=* -p gpu --gres=gpu:$0 -c $0  # Recommend allocating more CPU than, or equal to the number of GPU
+# note: the --max-jobs-run option is supported as a special case
+# by slurm.pl and you don't have to handle it in the config file.
diff --git a/egs2/mini_an4/enh_asr1/db.sh b/egs2/mini_an4/enh_asr1/db.sh
new file mode 120000
index 00000000000..3090b1bc350
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/db.sh
@@ -0,0 +1 @@
+../../TEMPLATE/enh_asr1/db.sh
\ No newline at end of file
diff --git a/egs2/mini_an4/enh_asr1/downloads.tar.gz b/egs2/mini_an4/enh_asr1/downloads.tar.gz
new file mode 120000
index 00000000000..e916a05f1df
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/downloads.tar.gz
@@ -0,0 +1 @@
+../../../egs/mini_an4/asr1/downloads.tar.gz
\ No newline at end of file
diff --git a/egs2/mini_an4/enh_asr1/enh_asr.sh b/egs2/mini_an4/enh_asr1/enh_asr.sh
new file mode 120000
index 00000000000..b00d9b13ef7
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/enh_asr.sh
@@ -0,0 +1 @@
+../../TEMPLATE/enh_asr1/enh_asr.sh
\ No newline at end of file
diff --git a/egs2/mini_an4/enh_asr1/local/data.sh b/egs2/mini_an4/enh_asr1/local/data.sh
new file mode 100755
index 00000000000..4b35a1668e0
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/local/data.sh
@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+log() {
+    local fname=${BASH_SOURCE[1]##*/}
+    echo -e "$(date '+%Y-%m-%dT%H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
+}
+SECONDS=0
+
+stage=1
+stop_stage=100
+
+an4_root=./downloads/an4
+
+log "$0 $*"
+. utils/parse_options.sh
+
+if [ $# -ne 0 ]; then
+    log "Error: No positional arguments are required."
+    exit 2
+fi
+
+. ./path.sh
+. ./cmd.sh
+
+train_set="train_nodev"
+train_dev="train_dev"
+
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    log "stage 1: Untar downloads.tar.gz"
+    if [ ! -e downloads/ ]; then
+        tar -xvf downloads.tar.gz
+    fi
+fi
+
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+    log "stage 2: Data preparation"
+    mkdir -p data/{train,test}
+
+    if [ ! -f ${an4_root}/README ]; then
+        echo Cannot find an4 root! Exiting...
+        exit 1
+    fi
+
+    python3 local/data_prep.py ${an4_root} sph2pipe
+
+    for x in test train; do
+        for f in text wav.scp utt2spk; do
+            sort data/${x}/${f} -o data/${x}/${f}
+        done
+        utils/utt2spk_to_spk2utt.pl data/${x}/utt2spk > data/${x}/spk2utt
+    done
+
+    # make a dev set
+    utils/subset_data_dir.sh --first data/train 1 data/${train_dev}
+    n=$(($(wc -l < data/train/text) - 1))
+    utils/subset_data_dir.sh --last data/train ${n} data/${train_set}
+
+    # Create "test_seg" in order to test the use case of segments
+    rm -rf data/test_seg
+    utils/copy_data_dir.sh data/test data/test_seg
+    <data/test/wav.scp awk '{ for(i=2;i<=NF;i++){a=a " " $i}; print($1 "_org", a) }' > data/test_seg/wav.scp
+    cat << EOF > data/test_seg/segments
+fcaw-cen8-b fcaw-cen8-b_org 0.0 2.9
+mmxg-cen8-b mmxg-cen8-b_org 0.0 2.3
+EOF
+
+    # for enh task
+    for x in test ${train_set} ${train_dev} test test_seg; do
+        for f in wav.scp text utt2spk segments; do
+            [ ! -f data/${x}/${f} ] && continue
+            mv data/${x}/${f} data/${x}/${f}.old
+            if [ $f = "segments" ]; then
+                <data/${x}/${f}.old awk '{$1=$1"_SIMU"; $2=$2"_SIMU"; print($0)}' > data/${x}/${f}
+            else
+                <data/${x}/${f}.old awk '{$1=$1"_SIMU"; print($0)}' > data/${x}/${f}
+            fi
+            rm data/${x}/${f}.old
+        done
+        utils/utt2spk_to_spk2utt.pl data/${x}/utt2spk > data/${x}/spk2utt
+
+        cp data/${x}/wav.scp data/${x}/spk1.scp
+        <data/${x}/wav.scp awk '{print($1, "SIMU")}' > data/${x}/utt2category
+    done
+fi
+
+
+log "Successfully finished. [elapsed=${SECONDS}s]"
diff --git a/egs2/mini_an4/enh_asr1/local/data_prep.py b/egs2/mini_an4/enh_asr1/local/data_prep.py
new file mode 120000
index 00000000000..d416349ede8
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/local/data_prep.py
@@ -0,0 +1 @@
+../../../../egs/an4/asr1/local/data_prep.py
\ No newline at end of file
diff --git a/egs2/mini_an4/enh_asr1/local/download_and_untar.sh b/egs2/mini_an4/enh_asr1/local/download_and_untar.sh
new file mode 120000
index 00000000000..40bf437ab02
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/local/download_and_untar.sh
@@ -0,0 +1 @@
+../../../../egs/an4/asr1/local/download_and_untar.sh
\ No newline at end of file
diff --git a/egs2/mini_an4/enh_asr1/local/path.sh b/egs2/mini_an4/enh_asr1/local/path.sh
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/egs2/mini_an4/enh_asr1/path.sh b/egs2/mini_an4/enh_asr1/path.sh
new file mode 120000
index 00000000000..f2720c6899b
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/path.sh
@@ -0,0 +1 @@
+../../TEMPLATE/enh_asr1/path.sh
\ No newline at end of file
diff --git a/egs2/mini_an4/enh_asr1/pyscripts b/egs2/mini_an4/enh_asr1/pyscripts
new file mode 120000
index 00000000000..008f9bd4bc5
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/pyscripts
@@ -0,0 +1 @@
+../../TEMPLATE/enh_asr1/pyscripts
\ No newline at end of file
diff --git a/egs2/mini_an4/enh_asr1/run.sh b/egs2/mini_an4/enh_asr1/run.sh
new file mode 100755
index 00000000000..7b01cd09b87
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/run.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+./enh_asr.sh \
+    --lang en \
+    --train_set train_nodev \
+    --valid_set train_dev \
+    --test_sets "train_dev test test_seg" \
+    --lm_train_text "data/train_nodev/text" "$@"
diff --git a/egs2/mini_an4/enh_asr1/scripts b/egs2/mini_an4/enh_asr1/scripts
new file mode 120000
index 00000000000..6c0f28ef23c
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/scripts
@@ -0,0 +1 @@
+../../TEMPLATE/enh_asr1/scripts
\ No newline at end of file
diff --git a/egs2/mini_an4/enh_asr1/steps b/egs2/mini_an4/enh_asr1/steps
new file mode 120000
index 00000000000..91f2d234e20
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/steps
@@ -0,0 +1 @@
+../../../tools/kaldi/egs/wsj/s5/steps
\ No newline at end of file
diff --git a/egs2/mini_an4/enh_asr1/utils b/egs2/mini_an4/enh_asr1/utils
new file mode 120000
index 00000000000..f49247da827
--- /dev/null
+++ b/egs2/mini_an4/enh_asr1/utils
@@ -0,0 +1 @@
+../../../tools/kaldi/egs/wsj/s5/utils
\ No newline at end of file
diff --git a/espnet2/enh/espnet_enh_s2t_model.py b/espnet2/enh/espnet_enh_s2t_model.py
index 25b56a6952e..a966f88a98b 100644
--- a/espnet2/enh/espnet_enh_s2t_model.py
+++ b/espnet2/enh/espnet_enh_s2t_model.py
@@ -46,6 +46,9 @@ def __init__(
 
         self.permutation_by_enh = permutation_by_enh
         self.calc_enh_loss = calc_enh_loss
+        self.extract_feats_in_collect_stats = (
+            self.s2t_model.extract_feats_in_collect_stats
+        )
 
     def forward(
         self,
diff --git a/test/espnet2/bin/test_asr_inference.py b/test/espnet2/bin/test_asr_inference.py
index 43bc230dcf6..7e7a18d9a1e 100644
--- a/test/espnet2/bin/test_asr_inference.py
+++ b/test/espnet2/bin/test_asr_inference.py
@@ -147,7 +147,7 @@ def test_EnhS2T_Speech2Text(enh_asr_config_file, lm_config_file):
         beam_size=1,
         enh_s2t_task=True,
     )
-    speech = np.random.randn(100000)
+    speech = np.random.randn(48000)
     results = speech2text(speech)
     for text, token, token_int, hyp in results:
         assert isinstance(text, str)

From 57ef7be58afb50ca799382b62eeb063cda6bd9cd Mon Sep 17 00:00:00 2001
From: YushiUeda <y.ueda1205@gmail.com>
Date: Sun, 24 Apr 2022 22:49:24 -0400
Subject: [PATCH 44/56] add harpervalley

---
 egs2/README.md                                |   1 +
 egs2/TEMPLATE/asr1/db.sh                      |   1 +
 egs2/harpervalley/asr1/README.md              |  28 ++++
 egs2/harpervalley/asr1/asr.sh                 |   1 +
 egs2/harpervalley/asr1/cmd.sh                 | 110 +++++++++++++++
 egs2/harpervalley/asr1/conf/decode_asr.yaml   |   1 +
 egs2/harpervalley/asr1/conf/fbank.conf        |   2 +
 egs2/harpervalley/asr1/conf/pbs.conf          |  11 ++
 egs2/harpervalley/asr1/conf/pitch.conf        |   1 +
 egs2/harpervalley/asr1/conf/queue.conf        |  12 ++
 egs2/harpervalley/asr1/conf/slurm.conf        |  14 ++
 egs2/harpervalley/asr1/conf/train_asr.yaml    |   1 +
 .../asr1/conf/tuning/train_asr.yaml           |  49 +++++++
 .../asr1/conf/tuning/train_asr_hubert.yaml    |  85 ++++++++++++
 egs2/harpervalley/asr1/db.sh                  |   1 +
 egs2/harpervalley/asr1/local/data.sh          |  74 +++++++++++
 egs2/harpervalley/asr1/local/data_prep.py     |  97 ++++++++++++++
 egs2/harpervalley/asr1/local/path.sh          |   0
 egs2/harpervalley/asr1/local/score.py         |   1 +
 egs2/harpervalley/asr1/local/score.sh         |   1 +
 egs2/harpervalley/asr1/local/split_data.py    | 125 ++++++++++++++++++
 egs2/harpervalley/asr1/path.sh                |   1 +
 egs2/harpervalley/asr1/pyscripts              |   1 +
 egs2/harpervalley/asr1/run.sh                 |  28 ++++
 egs2/harpervalley/asr1/scripts                |   1 +
 egs2/harpervalley/asr1/steps                  |   1 +
 egs2/harpervalley/asr1/utils                  |   1 +
 27 files changed, 649 insertions(+)
 create mode 100644 egs2/harpervalley/asr1/README.md
 create mode 120000 egs2/harpervalley/asr1/asr.sh
 create mode 100644 egs2/harpervalley/asr1/cmd.sh
 create mode 100644 egs2/harpervalley/asr1/conf/decode_asr.yaml
 create mode 100644 egs2/harpervalley/asr1/conf/fbank.conf
 create mode 100644 egs2/harpervalley/asr1/conf/pbs.conf
 create mode 100644 egs2/harpervalley/asr1/conf/pitch.conf
 create mode 100644 egs2/harpervalley/asr1/conf/queue.conf
 create mode 100644 egs2/harpervalley/asr1/conf/slurm.conf
 create mode 120000 egs2/harpervalley/asr1/conf/train_asr.yaml
 create mode 100644 egs2/harpervalley/asr1/conf/tuning/train_asr.yaml
 create mode 100644 egs2/harpervalley/asr1/conf/tuning/train_asr_hubert.yaml
 create mode 120000 egs2/harpervalley/asr1/db.sh
 create mode 100755 egs2/harpervalley/asr1/local/data.sh
 create mode 100755 egs2/harpervalley/asr1/local/data_prep.py
 create mode 100644 egs2/harpervalley/asr1/local/path.sh
 create mode 120000 egs2/harpervalley/asr1/local/score.py
 create mode 120000 egs2/harpervalley/asr1/local/score.sh
 create mode 100644 egs2/harpervalley/asr1/local/split_data.py
 create mode 120000 egs2/harpervalley/asr1/path.sh
 create mode 120000 egs2/harpervalley/asr1/pyscripts
 create mode 100755 egs2/harpervalley/asr1/run.sh
 create mode 120000 egs2/harpervalley/asr1/scripts
 create mode 120000 egs2/harpervalley/asr1/steps
 create mode 120000 egs2/harpervalley/asr1/utils

diff --git a/egs2/README.md b/egs2/README.md
index 185a20b875d..b44b29ae83a 100755
--- a/egs2/README.md
+++ b/egs2/README.md
@@ -38,6 +38,7 @@ See: https://espnet.github.io/espnet/espnet2_tutorial.html#recipes-using-espnet2
 | fsc_challenge           | Fluent Speech Commands Dataset MASE Eval Challenge splits                                                                        | SLU                     | ENG                   | https://github.com/maseEval/mase                                                                             |              |
 | gigaspeech              | GigaSpeech: An Evolving, Multi-domain ASR Corpus with 10,000 Hours of Transcribed Audio                                          | ASR                     | ENG                   | https://github.com/SpeechColab/GigaSpeech                                                                    |              |
 | grabo                   | Grabo dataset                                                                                                                    | SLU                     | ENG + NLD             | https://www.esat.kuleuven.be/psi/spraak/downloads/                                                           |              |
+| harpervalley             | HarperValleyBank: A Domain-Specific Spoken Dialog Corpus                                                                            | SLU                     | ENG                   | https://github.com/cricketclub/gridspace-stanford-harper-valley                                                       |              |
 | hkust                   | HKUST/MTS: A very large scale Mandarin telephone speech corpus                                                                   | ASR                     | CMN                   | https://catalog.ldc.upenn.edu/LDC2005S15                                                                     |              |
 | hui_acg                 | HUI-audio-corpus-german                                                                                                          | TTS                     | DEU                   | https://opendata.iisys.de/datasets.html#hui-audio-corpus-german                                              |              |
 | how2                    | How2: A Large-scale Dataset for Multimodal Language Understanding                                                                | ASR/MT/ST               | ENG->POR              | https://github.com/srvk/how2-dataset                                                                         |              |
diff --git a/egs2/TEMPLATE/asr1/db.sh b/egs2/TEMPLATE/asr1/db.sh
index e71e5587259..bbc9165498e 100755
--- a/egs2/TEMPLATE/asr1/db.sh
+++ b/egs2/TEMPLATE/asr1/db.sh
@@ -139,6 +139,7 @@ MALAYALAM=downloads
 ST_CMDS=downloads
 MS_INDIC_IS18=
 MARATHI=downloads
+HARPERVALLEY=downloads
 
 # For only CMU TIR environment
 if [[ "$(hostname)" == tir* ]]; then
diff --git a/egs2/harpervalley/asr1/README.md b/egs2/harpervalley/asr1/README.md
new file mode 100644
index 00000000000..e40e1a41589
--- /dev/null
+++ b/egs2/harpervalley/asr1/README.md
@@ -0,0 +1,28 @@
+<!-- Generated by scripts/utils/show_asr_result.sh -->
+# RESULTS
+## Environments
+- date: `Sat Apr 23 12:20:53 EDT 2022`
+- python version: `3.7.11 (default, Jul 27 2021, 14:32:16)  [GCC 7.5.0]`
+- espnet version: `espnet 0.10.7a1`
+- pytorch version: `pytorch 1.9.0+cu102`
+- Git hash: `0b1d15ebe0c36efcdf06d1b2e32361e3c8846cf6`
+  - Commit date: `Tue Apr 5 10:46:40 2022 -0400`
+- Pretrained Model: https://huggingface.co/espnet/YushiUeda_harpervalley_train_asr_hubert_raw_en_word
+
+## asr_train_asr_raw_en_word
+- config [conf/tuning/train_asr.yaml](conf/tuning/train_asr.yaml)
+- token_type word
+
+|dataset|Snt|Intent Classification (%)|
+|---|---|---|
+|inference_asr_model_valid.acc.ave_10best/test|1028|37.9|
+|inference_asr_model_valid.acc.ave_10best/valid|1430|35.5|
+
+## asr_train_asr_hubert_raw_en_word
+- config [conf/tuning/train_asr_hubert.yaml](conf/tuning/train_asr_hubert.yaml)
+- token_type word
+
+|dataset|Snt|Intent Classification (%)|
+|---|---|---|
+|inference_asr_model_valid.acc.ave_10best/test|1028|46.0|
+|inference_asr_model_valid.acc.ave_10best/valid|1430|46.6|
diff --git a/egs2/harpervalley/asr1/asr.sh b/egs2/harpervalley/asr1/asr.sh
new file mode 120000
index 00000000000..60b05122cfd
--- /dev/null
+++ b/egs2/harpervalley/asr1/asr.sh
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/asr.sh
\ No newline at end of file
diff --git a/egs2/harpervalley/asr1/cmd.sh b/egs2/harpervalley/asr1/cmd.sh
new file mode 100644
index 00000000000..2aae6919fef
--- /dev/null
+++ b/egs2/harpervalley/asr1/cmd.sh
@@ -0,0 +1,110 @@
+# ====== About run.pl, queue.pl, slurm.pl, and ssh.pl ======
+# Usage: <cmd>.pl [options] JOB=1:<nj> <log> <command...>
+# e.g.
+#   run.pl --mem 4G JOB=1:10 echo.JOB.log echo JOB
+#
+# Options:
+#   --time <time>: Limit the maximum time to execute.
+#   --mem <mem>: Limit the maximum memory usage.
+#   -–max-jobs-run <njob>: Limit the number parallel jobs. This is ignored for non-array jobs.
+#   --num-threads <ngpu>: Specify the number of CPU core.
+#   --gpu <ngpu>: Specify the number of GPU devices.
+#   --config: Change the configuration file from default.
+#
+# "JOB=1:10" is used for "array jobs" and it can control the number of parallel jobs.
+# The left string of "=", i.e. "JOB", is replaced by <N>(Nth job) in the command and the log file name,
+# e.g. "echo JOB" is changed to "echo 3" for the 3rd job and "echo 8" for 8th job respectively.
+# Note that the number must start with a positive number, so you can't use "JOB=0:10" for example.
+#
+# run.pl, queue.pl, slurm.pl, and ssh.pl have unified interface, not depending on its backend.
+# These options are mapping to specific options for each backend and
+# it is configured by "conf/queue.conf" and "conf/slurm.conf" by default.
+# If jobs failed, your configuration might be wrong for your environment.
+#
+#
+# The official documentation for run.pl, queue.pl, slurm.pl, and ssh.pl:
+#   "Parallelization in Kaldi": http://kaldi-asr.org/doc/queue.html
+# =========================================================~
+
+
+# Select the backend used by run.sh from "local", "stdout", "sge", "slurm", or "ssh"
+cmd_backend='local'
+
+# Local machine, without any Job scheduling system
+if [ "${cmd_backend}" = local ]; then
+
+    # The other usage
+    export train_cmd="run.pl"
+    # Used for "*_train.py": "--gpu" is appended optionally by run.sh
+    export cuda_cmd="run.pl"
+    # Used for "*_recog.py"
+    export decode_cmd="run.pl"
+
+# Local machine logging to stdout and log file, without any Job scheduling system
+elif [ "${cmd_backend}" = stdout ]; then
+
+    # The other usage
+    export train_cmd="stdout.pl"
+    # Used for "*_train.py": "--gpu" is appended optionally by run.sh
+    export cuda_cmd="stdout.pl"
+    # Used for "*_recog.py"
+    export decode_cmd="stdout.pl"
+
+
+# "qsub" (Sun Grid Engine, or derivation of it)
+elif [ "${cmd_backend}" = sge ]; then
+    # The default setting is written in conf/queue.conf.
+    # You must change "-q g.q" for the "queue" for your environment.
+    # To know the "queue" names, type "qhost -q"
+    # Note that to use "--gpu *", you have to setup "complex_value" for the system scheduler.
+
+    export train_cmd="queue.pl"
+    export cuda_cmd="queue.pl"
+    export decode_cmd="queue.pl"
+
+
+# "qsub" (Torque/PBS.)
+elif [ "${cmd_backend}" = pbs ]; then
+    # The default setting is written in conf/pbs.conf.
+
+    export train_cmd="pbs.pl"
+    export cuda_cmd="pbs.pl"
+    export decode_cmd="pbs.pl"
+
+
+# "sbatch" (Slurm)
+elif [ "${cmd_backend}" = slurm ]; then
+    # The default setting is written in conf/slurm.conf.
+    # You must change "-p cpu" and "-p gpu" for the "partition" for your environment.
+    # To know the "partion" names, type "sinfo".
+    # You can use "--gpu * " by default for slurm and it is interpreted as "--gres gpu:*"
+    # The devices are allocated exclusively using "${CUDA_VISIBLE_DEVICES}".
+
+    export train_cmd="slurm.pl"
+    export cuda_cmd="slurm.pl"
+    export decode_cmd="slurm.pl"
+
+elif [ "${cmd_backend}" = ssh ]; then
+    # You have to create ".queue/machines" to specify the host to execute jobs.
+    # e.g. .queue/machines
+    #   host1
+    #   host2
+    #   host3
+    # Assuming you can login them without any password, i.e. You have to set ssh keys.
+
+    export train_cmd="ssh.pl"
+    export cuda_cmd="ssh.pl"
+    export decode_cmd="ssh.pl"
+
+# This is an example of specifying several unique options in the JHU CLSP cluster setup.
+# Users can modify/add their own command options according to their cluster environments.
+elif [ "${cmd_backend}" = jhu ]; then
+
+    export train_cmd="queue.pl --mem 2G"
+    export cuda_cmd="queue-freegpu.pl --mem 2G --gpu 1 --config conf/queue.conf"
+    export decode_cmd="queue.pl --mem 4G"
+
+else
+    echo "$0: Error: Unknown cmd_backend=${cmd_backend}" 1>&2
+    return 1
+fi
diff --git a/egs2/harpervalley/asr1/conf/decode_asr.yaml b/egs2/harpervalley/asr1/conf/decode_asr.yaml
new file mode 100644
index 00000000000..e2408c35e82
--- /dev/null
+++ b/egs2/harpervalley/asr1/conf/decode_asr.yaml
@@ -0,0 +1 @@
+beam_size: 1
\ No newline at end of file
diff --git a/egs2/harpervalley/asr1/conf/fbank.conf b/egs2/harpervalley/asr1/conf/fbank.conf
new file mode 100644
index 00000000000..82ac7bd0dbc
--- /dev/null
+++ b/egs2/harpervalley/asr1/conf/fbank.conf
@@ -0,0 +1,2 @@
+--sample-frequency=16000 
+--num-mel-bins=80
diff --git a/egs2/harpervalley/asr1/conf/pbs.conf b/egs2/harpervalley/asr1/conf/pbs.conf
new file mode 100644
index 00000000000..119509938ce
--- /dev/null
+++ b/egs2/harpervalley/asr1/conf/pbs.conf
@@ -0,0 +1,11 @@
+# Default configuration
+command qsub -V -v PATH -S /bin/bash
+option name=* -N $0
+option mem=* -l mem=$0
+option mem=0          # Do not add anything to qsub_opts
+option num_threads=* -l ncpus=$0
+option num_threads=1  # Do not add anything to qsub_opts
+option num_nodes=* -l nodes=$0:ppn=1
+default gpu=0
+option gpu=0
+option gpu=* -l ngpus=$0
diff --git a/egs2/harpervalley/asr1/conf/pitch.conf b/egs2/harpervalley/asr1/conf/pitch.conf
new file mode 100644
index 00000000000..e959a19d5b8
--- /dev/null
+++ b/egs2/harpervalley/asr1/conf/pitch.conf
@@ -0,0 +1 @@
+--sample-frequency=16000
diff --git a/egs2/harpervalley/asr1/conf/queue.conf b/egs2/harpervalley/asr1/conf/queue.conf
new file mode 100644
index 00000000000..500582fab31
--- /dev/null
+++ b/egs2/harpervalley/asr1/conf/queue.conf
@@ -0,0 +1,12 @@
+# Default configuration
+command qsub -v PATH -cwd -S /bin/bash -j y -l arch=*64*
+option name=* -N $0
+option mem=* -l mem_free=$0,ram_free=$0
+option mem=0          # Do not add anything to qsub_opts
+option num_threads=* -pe smp $0
+option num_threads=1  # Do not add anything to qsub_opts
+option max_jobs_run=* -tc $0
+option num_nodes=* -pe mpi $0  # You must set this PE as allocation_rule=1
+default gpu=0
+option gpu=0
+option gpu=* -l gpu=$0 -q g.q
diff --git a/egs2/harpervalley/asr1/conf/slurm.conf b/egs2/harpervalley/asr1/conf/slurm.conf
new file mode 100644
index 00000000000..3b229673638
--- /dev/null
+++ b/egs2/harpervalley/asr1/conf/slurm.conf
@@ -0,0 +1,14 @@
+# Default configuration
+command sbatch --export=PATH
+option name=* --job-name $0
+option time=* --time $0
+option mem=* --mem-per-cpu $0
+option mem=0
+option num_threads=* --cpus-per-task $0
+option num_threads=1 --cpus-per-task 1
+option num_nodes=* --nodes $0
+default gpu=0
+option gpu=0 -p cpu
+option gpu=* -p gpu --gres=gpu:$0 -c $0  # Recommend allocating more CPU than, or equal to the number of GPU
+# note: the --max-jobs-run option is supported as a special case
+# by slurm.pl and you don't have to handle it in the config file.
diff --git a/egs2/harpervalley/asr1/conf/train_asr.yaml b/egs2/harpervalley/asr1/conf/train_asr.yaml
new file mode 120000
index 00000000000..a7463d73f0e
--- /dev/null
+++ b/egs2/harpervalley/asr1/conf/train_asr.yaml
@@ -0,0 +1 @@
+tuning/train_asr.yaml
\ No newline at end of file
diff --git a/egs2/harpervalley/asr1/conf/tuning/train_asr.yaml b/egs2/harpervalley/asr1/conf/tuning/train_asr.yaml
new file mode 100644
index 00000000000..ac943f876ac
--- /dev/null
+++ b/egs2/harpervalley/asr1/conf/tuning/train_asr.yaml
@@ -0,0 +1,49 @@
+# network architecture
+# encoder related
+encoder: transformer
+encoder_conf:
+    output_size: 256    # dimension of attention
+    attention_heads: 4
+    linear_units: 2048  # the number of units of position-wise feed forward
+    num_blocks: 12      # the number of encoder blocks
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.0
+    input_layer: conv2d # encoder architecture type
+    normalize_before: true
+
+# decoder related
+decoder: transformer
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.0
+    src_attention_dropout_rate: 0.0
+optim: adam
+optim_conf:
+    lr: 0.0002
+scheduler: warmuplr     # pytorch v1.1.0+ required
+scheduler_conf:
+    warmup_steps: 25000
+max_epoch: 50
+keep_nbest_models: 10
+
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 40
+    num_time_mask: 2
+    
\ No newline at end of file
diff --git a/egs2/harpervalley/asr1/conf/tuning/train_asr_hubert.yaml b/egs2/harpervalley/asr1/conf/tuning/train_asr_hubert.yaml
new file mode 100644
index 00000000000..49867ef89b7
--- /dev/null
+++ b/egs2/harpervalley/asr1/conf/tuning/train_asr_hubert.yaml
@@ -0,0 +1,85 @@
+# network architecture
+# encoder related
+encoder: conformer
+encoder_conf:
+    output_size: 512
+    attention_heads: 8
+    linear_units: 2048
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d
+    normalize_before: true
+    macaron_style: true
+    pos_enc_layer_type: "rel_pos"
+    selfattention_layer_type: "rel_selfattn"
+    activation_type: "swish"
+    use_cnn_module:  true
+    cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 8
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+optim: adam
+optim_conf:
+    lr: 0.0002
+scheduler: warmuplr     # pytorch v1.1.0+ required #Tune warmup steps
+scheduler_conf:
+    warmup_steps: 25000
+max_epoch: 50
+
+freeze_param: [
+"frontend.upstream"
+]
+
+frontend_conf:
+  n_fft: 512
+  hop_length: 256
+
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream: hubert_large_ll60k  # Note: If the upstream is changed, please change the input_size in the preencoder.
+    download_dir: ./hub
+    multilayer_feature: True
+
+preencoder: linear
+preencoder_conf:
+    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
+    output_size: 80
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+    extract_feats_in_collect_stats: false   # Note: "False" means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend.
+
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 40
+    num_time_mask: 2
+
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
\ No newline at end of file
diff --git a/egs2/harpervalley/asr1/db.sh b/egs2/harpervalley/asr1/db.sh
new file mode 120000
index 00000000000..50d86130898
--- /dev/null
+++ b/egs2/harpervalley/asr1/db.sh
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/db.sh
\ No newline at end of file
diff --git a/egs2/harpervalley/asr1/local/data.sh b/egs2/harpervalley/asr1/local/data.sh
new file mode 100755
index 00000000000..a844a7ff761
--- /dev/null
+++ b/egs2/harpervalley/asr1/local/data.sh
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+log() {
+    local fname=${BASH_SOURCE[1]##*/}
+    echo -e "$(date '+%Y-%m-%dT%H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
+}
+SECONDS=0
+
+
+stage=1
+stop_stage=100000
+fs=16k
+log "$0 $*"
+. utils/parse_options.sh
+
+. ./db.sh
+. ./path.sh
+. ./cmd.sh
+
+if [ $# -ne 0 ]; then
+    log "Error: No positional arguments are required."
+    exit 2
+fi
+
+if [ -z "${HARPERVALLEY}" ]; then
+    log "Fill the value of 'HARPERVALLEY' of db.sh"
+    exit 1
+fi
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    if [ ! -e "${HARPERVALLEY}/LICENSE" ]; then
+        echo "stage 1: Download data to ${HARPERVALLEY}"
+        mkdir -p ${HARPERVALLEY}
+        git clone https://github.com/cricketclub/gridspace-stanford-harper-valley.git ${HARPERVALLEY}
+    else
+        log "stage 1: ${HARPERVALLEY}/LICENSE is already existing. Skip data downloading"
+    fi
+fi
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+    log "stage2 : Data Preparation"
+    if [ -n "$(ls data/tmp/)" ]; then
+        rm -r data/tmp/
+    fi
+    for file in ${HARPERVALLEY}/data/transcript/*.json; do
+        filename=$(basename "${file%.*}")
+        dirname="${HARPERVALLEY}/data/"
+        python3 local/data_prep.py --source_dir "$dirname" \
+            --audio_dir "data/audio" \
+            --filename "$filename" \
+            --target_dir "data/tmp" \
+            --min_length 4
+    done
+    sed -i -e 's/<unk>/\[unk\]/g' data/tmp/text
+    mkdir -p data/{train,valid,test}
+    python3 local/split_data.py --source_dir "data/tmp" \
+        --min_spk_utt 10 \
+        --train_frac 0.8 \
+        --val_frac 0.1
+    for x in test valid train; do
+        for f in text wav.scp utt2spk segments; do
+            sort data/${x}/${f} -o data/${x}/${f}
+        done
+        utils/utt2spk_to_spk2utt.pl data/${x}/utt2spk > "data/${x}/spk2utt"
+        utils/validate_data_dir.sh --no-feats data/${x} || exit 1
+    done
+fi
+
+log "Successfully finished. [elapsed=${SECONDS}s]"
diff --git a/egs2/harpervalley/asr1/local/data_prep.py b/egs2/harpervalley/asr1/local/data_prep.py
new file mode 100755
index 00000000000..7e404ad2249
--- /dev/null
+++ b/egs2/harpervalley/asr1/local/data_prep.py
@@ -0,0 +1,97 @@
+import argparse
+import json
+import os
+import sys
+import wave
+
+
+def load_json(f_path):
+    with open(f_path, "r") as f:
+        return json.load(f)
+
+
+def process_data(target_dir, source_dir, audio_dir, filename, min_length):
+    if not os.path.exists(target_dir):
+        os.makedirs(target_dir)
+
+    agent_wav_path = os.path.join(audio_dir, "agent", filename + ".wav")
+    caller_wav_path = os.path.join(audio_dir, "caller", filename + ".wav")
+    # exit if the wav files do not exist.
+    if not os.path.isfile(agent_wav_path) or not os.path.isfile(agent_wav_path):
+        sys.exit()
+
+    with wave.open(agent_wav_path, "rb") as wa, wave.open(caller_wav_path, "rb") as wc:
+        wa_length = wa.getnframes() / wa.getframerate()
+        wc_length = wc.getnframes() / wc.getframerate()
+
+    with open(
+        os.path.join(target_dir, "wav.scp"), "a", encoding="utf-8"
+    ) as wavscp, open(
+        os.path.join(target_dir, "utt2spk"), "a", encoding="utf-8"
+    ) as utt2spk, open(
+        os.path.join(target_dir, "segments"), "a", encoding="utf-8"
+    ) as segments, open(
+        os.path.join(target_dir, "text"), "a", encoding="utf-8"
+    ) as text:
+
+        metadata_f = load_json(os.path.join(source_dir, "metadata", filename + ".json"))
+        transcript_f = load_json(
+            os.path.join(source_dir, "transcript", filename + ".json")
+        )
+
+        agent_spk_id = metadata_f["agent"]["speaker_id"]
+        caller_spk_id = metadata_f["caller"]["speaker_id"]
+        task_type = metadata_f["tasks"][0]["task_type"].replace(" ", "_")
+        agent_rec_id = "{}-{}".format(agent_spk_id, filename)
+        caller_rec_id = "{}-{}".format(caller_spk_id, filename)
+
+        agent_utt_num = 0
+        caller_utt_num = 0
+        for v in transcript_f:
+            transcript = v["human_transcript"]
+            # Throw away utterances with < min_length words or 100 ms
+            if len(transcript.split()) < min_length or int(v["duration_ms"]) < 100:
+                continue
+            begin_ms = int(v["offset_ms"])
+            end_ms = begin_ms + int(v["duration_ms"])
+            begin_sec = begin_ms / 1000
+            end_sec = end_ms / 1000
+            if v["speaker_role"] == "agent":
+                if end_sec > wa_length:
+                    continue
+                utt_id = "{}_{}_{}".format(agent_rec_id, begin_ms, end_ms)
+                utt2spk.write("{} {}\n".format(utt_id, agent_spk_id))
+                segments.write(
+                    "{} {} {} {}\n".format(utt_id, agent_rec_id, begin_sec, end_sec)
+                )
+                agent_utt_num += 1
+            else:
+                if end_sec > wc_length:
+                    continue
+                utt_id = "{}_{}_{}".format(caller_rec_id, begin_ms, end_ms)
+                utt2spk.write("{} {}\n".format(utt_id, caller_spk_id))
+                segments.write(
+                    "{} {} {} {}\n".format(utt_id, caller_rec_id, begin_sec, end_sec)
+                )
+                caller_utt_num += 1
+            text.write("{} {} {}\n".format(utt_id, task_type, transcript))
+
+        # write wav.scp only if utterances exist
+        if agent_utt_num > 0:
+            wavscp.write("{} {}\n".format(agent_rec_id, agent_wav_path))
+        if caller_utt_num > 0:
+            wavscp.write("{} {}\n".format(caller_rec_id, caller_wav_path))
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--target_dir", type=str, default="data/tmp")
+parser.add_argument("--source_dir", type=str, required=True, help="Path to source data")
+parser.add_argument("--audio_dir", type=str, required=True, help="Path to audio data")
+parser.add_argument("--filename", type=str, required=True, help="filename")
+parser.add_argument("--min_length", type=int, default=4)
+
+args = parser.parse_args()
+
+process_data(
+    args.target_dir, args.source_dir, args.audio_dir, args.filename, args.min_length
+)
diff --git a/egs2/harpervalley/asr1/local/path.sh b/egs2/harpervalley/asr1/local/path.sh
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/egs2/harpervalley/asr1/local/score.py b/egs2/harpervalley/asr1/local/score.py
new file mode 120000
index 00000000000..673506176eb
--- /dev/null
+++ b/egs2/harpervalley/asr1/local/score.py
@@ -0,0 +1 @@
+../../../slurp/asr1/local/score.py
\ No newline at end of file
diff --git a/egs2/harpervalley/asr1/local/score.sh b/egs2/harpervalley/asr1/local/score.sh
new file mode 120000
index 00000000000..eee438437a6
--- /dev/null
+++ b/egs2/harpervalley/asr1/local/score.sh
@@ -0,0 +1 @@
+../../../slurp/asr1/local/score.sh
\ No newline at end of file
diff --git a/egs2/harpervalley/asr1/local/split_data.py b/egs2/harpervalley/asr1/local/split_data.py
new file mode 100644
index 00000000000..9b75ec72adb
--- /dev/null
+++ b/egs2/harpervalley/asr1/local/split_data.py
@@ -0,0 +1,125 @@
+import argparse
+import os
+import random
+
+
+def get_spk_list(utt2spk_path, min_spk_utt=10, train_frac=0.8, val_frac=0.1):
+    # collect spk_id and count
+    with open(utt2spk_path, "r", encoding="utf-8") as utt2spk:
+        spk_dict = {}
+        for line in utt2spk.readlines():
+            spk_id = line.strip().split()[1]
+            if spk_id in spk_dict:
+                spk_dict[spk_id] += 1
+            else:
+                spk_dict[spk_id] = 1
+
+    # create spk_list
+    spk_list = []
+    for spk_id, count in spk_dict.items():
+        if count > min_spk_utt:
+            spk_list.append(spk_id)
+
+    # split spk_list
+    spk_list = sorted(spk_list)
+    num_speaker = len(spk_list)
+    num_train = int(train_frac * num_speaker)
+    num_val = int(val_frac * num_speaker)
+    random.seed(42)
+    random.shuffle(spk_list)
+
+    train_spk_list = spk_list[:num_train]
+    dev_spk_list = spk_list[num_train : num_train + num_val]
+    test_spk_list = spk_list[num_train + num_val :]
+
+    return train_spk_list, dev_spk_list, test_spk_list
+
+
+def split_files(source_dir, min_spk_utt, train_frac, val_frac):
+    wavscp_train = open("data/train/wav.scp", "w", encoding="utf-8")
+    utt2spk_train = open("data/train/utt2spk", "w", encoding="utf-8")
+    segments_train = open("data/train/segments", "w", encoding="utf-8")
+    text_train = open("data/train/text", "w", encoding="utf-8")
+    wavscp_dev = open("data/valid/wav.scp", "w", encoding="utf-8")
+    utt2spk_dev = open("data/valid/utt2spk", "w", encoding="utf-8")
+    segments_dev = open("data/valid/segments", "w", encoding="utf-8")
+    text_dev = open("data/valid/text", "w", encoding="utf-8")
+    wavscp_test = open("data/test/wav.scp", "w", encoding="utf-8")
+    utt2spk_test = open("data/test/utt2spk", "w", encoding="utf-8")
+    segments_test = open("data/test/segments", "w", encoding="utf-8")
+    text_test = open("data/test/text", "w", encoding="utf-8")
+
+    train_spk_list, dev_spk_list, test_spk_list = get_spk_list(
+        os.path.join(source_dir, "utt2spk"), min_spk_utt, train_frac, val_frac
+    )
+
+    # split wav.scp
+    with open(os.path.join(source_dir, "wav.scp"), "r", encoding="utf-8") as wavscp:
+        for line in wavscp.readlines():
+            rec_id = line.strip().split()[0]
+            spk_id = rec_id.split("-")[0]
+            if spk_id in train_spk_list:
+                wavscp_train.write("{}\n".format(line.strip()))
+            elif spk_id in dev_spk_list:
+                wavscp_dev.write("{}\n".format(line.strip()))
+            elif spk_id in test_spk_list:
+                wavscp_test.write("{}\n".format(line.strip()))
+
+    # split utt2spk
+    with open(os.path.join(source_dir, "utt2spk"), "r", encoding="utf-8") as utt2spk:
+        for line in utt2spk.readlines():
+            spk_id = line.strip().split()[1]
+            if spk_id in train_spk_list:
+                utt2spk_train.write("{}\n".format(line.strip()))
+            elif spk_id in dev_spk_list:
+                utt2spk_dev.write("{}\n".format(line.strip()))
+            elif spk_id in test_spk_list:
+                utt2spk_test.write("{}\n".format(line.strip()))
+
+    # split segments
+    with open(os.path.join(source_dir, "segments"), "r", encoding="utf-8") as segments:
+        for line in segments.readlines():
+            utt_id = line.strip().split()[0]
+            spk_id = utt_id.split("-")[0]
+            if spk_id in train_spk_list:
+                segments_train.write("{}\n".format(line.strip()))
+            elif spk_id in dev_spk_list:
+                segments_dev.write("{}\n".format(line.strip()))
+            elif spk_id in test_spk_list:
+                segments_test.write("{}\n".format(line.strip()))
+
+    # split text
+    with open(os.path.join(source_dir, "text"), "r", encoding="utf-8") as text:
+        for line in text.readlines():
+            utt_id = line.strip().split()[0]
+            spk_id = utt_id.split("-")[0]
+            if spk_id in train_spk_list:
+                text_train.write("{}\n".format(line.strip()))
+            elif spk_id in dev_spk_list:
+                text_dev.write("{}\n".format(line.strip()))
+            elif spk_id in test_spk_list:
+                text_test.write("{}\n".format(line.strip()))
+
+    wavscp_train.close()
+    utt2spk_train.close()
+    segments_train.close()
+    text_train.close()
+    wavscp_dev.close()
+    utt2spk_dev.close()
+    segments_dev.close()
+    text_dev.close()
+    wavscp_test.close()
+    utt2spk_test.close()
+    segments_test.close()
+    text_test.close()
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--source_dir", type=str, default="data/tmp")
+parser.add_argument("--min_spk_utt", type=int, default=10)
+parser.add_argument("--train_frac", type=float, default=0.8)
+parser.add_argument("--val_frac", type=float, default=0.1)
+
+args = parser.parse_args()
+
+split_files(args.source_dir, args.min_spk_utt, args.train_frac, args.val_frac)
diff --git a/egs2/harpervalley/asr1/path.sh b/egs2/harpervalley/asr1/path.sh
new file mode 120000
index 00000000000..c9ac0a75bc6
--- /dev/null
+++ b/egs2/harpervalley/asr1/path.sh
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/path.sh
\ No newline at end of file
diff --git a/egs2/harpervalley/asr1/pyscripts b/egs2/harpervalley/asr1/pyscripts
new file mode 120000
index 00000000000..ac68ad75b60
--- /dev/null
+++ b/egs2/harpervalley/asr1/pyscripts
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/pyscripts
\ No newline at end of file
diff --git a/egs2/harpervalley/asr1/run.sh b/egs2/harpervalley/asr1/run.sh
new file mode 100755
index 00000000000..a35d91ec261
--- /dev/null
+++ b/egs2/harpervalley/asr1/run.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+train_set="train"
+valid_set="valid"
+test_sets="test valid"
+
+asr_config=conf/train_asr.yaml
+
+./asr.sh \
+    --lang en \
+    --ngpu 1 \
+    --use_lm false \
+    --nbpe 5000 \
+    --bpe_nlsyms "[unk]" \
+    --token_type word\
+    --audio_format flac\
+    --feats_type raw\
+    --max_wav_duration 30 \
+    --inference_asr_model valid.acc.ave_10best.pth\
+    --asr_config "${asr_config}" \
+    --train_set "${train_set}" \
+    --valid_set "${valid_set}" \
+    --test_sets "${test_sets}" "$@"
\ No newline at end of file
diff --git a/egs2/harpervalley/asr1/scripts b/egs2/harpervalley/asr1/scripts
new file mode 120000
index 00000000000..b25829705dc
--- /dev/null
+++ b/egs2/harpervalley/asr1/scripts
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/scripts
\ No newline at end of file
diff --git a/egs2/harpervalley/asr1/steps b/egs2/harpervalley/asr1/steps
new file mode 120000
index 00000000000..91f2d234e20
--- /dev/null
+++ b/egs2/harpervalley/asr1/steps
@@ -0,0 +1 @@
+../../../tools/kaldi/egs/wsj/s5/steps
\ No newline at end of file
diff --git a/egs2/harpervalley/asr1/utils b/egs2/harpervalley/asr1/utils
new file mode 120000
index 00000000000..f49247da827
--- /dev/null
+++ b/egs2/harpervalley/asr1/utils
@@ -0,0 +1 @@
+../../../tools/kaldi/egs/wsj/s5/utils
\ No newline at end of file

From 720f64757643fdb19346e2539ceac2a45b0d57ed Mon Sep 17 00:00:00 2001
From: Muqiao Yang <muqiaoy@andrew.cmu.edu>
Date: Sun, 24 Apr 2022 23:00:50 -0400
Subject: [PATCH 45/56] add convtasnet recipe for dns_ins20

---
 egs2/dns_ins20/enh1/README.md                 | 24 ++++++++
 .../conf/tuning/train_enh_conv_tasnet.yaml    | 60 +++++++++++++++++++
 2 files changed, 84 insertions(+)
 create mode 100644 egs2/dns_ins20/enh1/conf/tuning/train_enh_conv_tasnet.yaml

diff --git a/egs2/dns_ins20/enh1/README.md b/egs2/dns_ins20/enh1/README.md
index 1f2b00b41df..07d5a0d7117 100644
--- a/egs2/dns_ins20/enh1/README.md
+++ b/egs2/dns_ins20/enh1/README.md
@@ -44,3 +44,27 @@
 
 Note: Here, the model is only trained on data without reverberation.
 Note: Here, the PESQ score is calculated based on https://github.com/vBaiCai/python-pesq.
+
+
+<!-- Generated by ./scripts/utils/show_enh_score.sh -->
+# RESULTS
+## Environments
+- date: `Fri Feb 25 05:12:24 EST 2022`
+- python version: `3.8.12 (default, Oct 12 2021, 13:49:34)  [GCC 7.5.0]`
+- espnet version: `espnet 0.10.6a1`
+- pytorch version: `pytorch 1.10.2+cu102`
+- Git hash: `21c02c8f578b9860e6bf38c86a0bd7cd0412c7f8`
+  - Commit date: `Sun Feb 6 15:37:51 2022 -0500`
+
+
+## enh_train_enh_conv_tasnet_raw
+
+- config: ./conf/tuning/train_enh_conv_tasnet.yaml
+- model: https://huggingface.co/muqiaoy/muqiaoy_dns_ins20_enh_train_enh_conv_tasnet_raw
+
+
+| dataset                           | STOI | SAR   | SDR   | SIR  |
+| --------------------------------- | ---- | ----- | ----- | ---- |
+| enhanced_cv_synthetic             | 0.97 | 24.52 | 24.52 | 0.00 |
+| enhanced_tt_synthetic_no_reverb   | 0.96 | 17.66 | 17.66 | 0.00 |
+| enhanced_tt_synthetic_with_reverb | 0.84 | 11.84 | 11.84 | 0.00 |
diff --git a/egs2/dns_ins20/enh1/conf/tuning/train_enh_conv_tasnet.yaml b/egs2/dns_ins20/enh1/conf/tuning/train_enh_conv_tasnet.yaml
new file mode 100644
index 00000000000..266c711a641
--- /dev/null
+++ b/egs2/dns_ins20/enh1/conf/tuning/train_enh_conv_tasnet.yaml
@@ -0,0 +1,60 @@
+optim: adam
+init: xavier_uniform
+max_epoch: 100
+batch_type: folded
+batch_size: 16 # batch_size 16 can be trained on 4 RTX 2080ti
+iterator_type: chunk
+chunk_length: 40000
+num_workers: 4
+optim_conf:
+    lr: 1.0e-03
+    eps: 1.0e-08
+    weight_decay: 0
+patience: 4
+val_scheduler_criterion:
+- valid
+- loss
+best_model_criterion:
+-   - valid
+    - si_snr
+    - max
+-   - valid
+    - loss
+    - min
+keep_nbest_models: 1
+scheduler: reducelronplateau
+scheduler_conf:
+    mode: min
+    factor: 0.5
+    patience: 1
+encoder: conv
+encoder_conf:
+    channel: 256
+    kernel_size: 20
+    stride: 10
+decoder: conv
+decoder_conf:
+    channel: 256
+    kernel_size: 20
+    stride: 10
+separator: tcn
+separator_conf:
+    num_spk: 1
+    layer: 8
+    stack: 4
+    bottleneck_dim: 256
+    hidden_dim: 512
+    kernel: 3
+    causal: False
+    norm_type: "gLN"
+    nonlinear: relu
+
+criterions: 
+  # The first criterion
+  - name: si_snr 
+    conf:
+      eps: 1.0e-7
+    wrapper: pit
+    wrapper_conf:
+      weight: 1.0
+      independent_perm: True

From 9d5fedc3a24760629352b94312ce5d3c1022b119 Mon Sep 17 00:00:00 2001
From: YushiUeda <y.ueda1205@gmail.com>
Date: Sun, 24 Apr 2022 23:30:59 -0400
Subject: [PATCH 46/56] fix ci error

---
 egs2/harpervalley/asr1/local/data.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/egs2/harpervalley/asr1/local/data.sh b/egs2/harpervalley/asr1/local/data.sh
index a844a7ff761..a7221b2b6ed 100755
--- a/egs2/harpervalley/asr1/local/data.sh
+++ b/egs2/harpervalley/asr1/local/data.sh
@@ -14,7 +14,6 @@ SECONDS=0
 
 stage=1
 stop_stage=100000
-fs=16k
 log "$0 $*"
 . utils/parse_options.sh
 

From 6c8fd58d159b516fbc82e29e177f684839e3352e Mon Sep 17 00:00:00 2001
From: Muqiao Yang <muqiaoy@andrew.cmu.edu>
Date: Mon, 25 Apr 2022 00:02:59 -0400
Subject: [PATCH 47/56] Update README.md

---
 egs2/dns_ins20/enh1/README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/egs2/dns_ins20/enh1/README.md b/egs2/dns_ins20/enh1/README.md
index 07d5a0d7117..d64724bfd65 100644
--- a/egs2/dns_ins20/enh1/README.md
+++ b/egs2/dns_ins20/enh1/README.md
@@ -49,7 +49,7 @@ Note: Here, the PESQ score is calculated based on https://github.com/vBaiCai/pyt
 <!-- Generated by ./scripts/utils/show_enh_score.sh -->
 # RESULTS
 ## Environments
-- date: `Fri Feb 25 05:12:24 EST 2022`
+- date: `Sun Apr 24 23:59:01 EDT 2022`
 - python version: `3.8.12 (default, Oct 12 2021, 13:49:34)  [GCC 7.5.0]`
 - espnet version: `espnet 0.10.6a1`
 - pytorch version: `pytorch 1.10.2+cu102`
@@ -63,8 +63,8 @@ Note: Here, the PESQ score is calculated based on https://github.com/vBaiCai/pyt
 - model: https://huggingface.co/muqiaoy/muqiaoy_dns_ins20_enh_train_enh_conv_tasnet_raw
 
 
-| dataset                           | STOI | SAR   | SDR   | SIR  |
+| dataset                           | STOI | SAR   | SDR   | SI_SNR  |
 | --------------------------------- | ---- | ----- | ----- | ---- |
-| enhanced_cv_synthetic             | 0.97 | 24.52 | 24.52 | 0.00 |
-| enhanced_tt_synthetic_no_reverb   | 0.96 | 17.66 | 17.66 | 0.00 |
-| enhanced_tt_synthetic_with_reverb | 0.84 | 11.84 | 11.84 | 0.00 |
+| enhanced_cv_synthetic             | 0.97 | 24.52 | 24.52 | 24.43 |
+| enhanced_tt_synthetic_no_reverb   | 0.96 | 17.66 | 17.66 | 17.69 |
+| enhanced_tt_synthetic_with_reverb | 0.84 | 11.84 | 11.84 | 11.15 |

From d57e1729abcbd20608341d806194691749769d17 Mon Sep 17 00:00:00 2001
From: YushiUeda <y.ueda1205@gmail.com>
Date: Mon, 25 Apr 2022 00:07:14 -0400
Subject: [PATCH 48/56] fix ci error

---
 egs2/harpervalley/asr1/local/data.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/egs2/harpervalley/asr1/local/data.sh b/egs2/harpervalley/asr1/local/data.sh
index a7221b2b6ed..519dc20d399 100755
--- a/egs2/harpervalley/asr1/local/data.sh
+++ b/egs2/harpervalley/asr1/local/data.sh
@@ -46,7 +46,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
     if [ -n "$(ls data/tmp/)" ]; then
         rm -r data/tmp/
     fi
-    for file in ${HARPERVALLEY}/data/transcript/*.json; do
+    for file in "${HARPERVALLEY}"/data/transcript/*.json; do
         filename=$(basename "${file%.*}")
         dirname="${HARPERVALLEY}/data/"
         python3 local/data_prep.py --source_dir "$dirname" \

From bd85a97db8c93f47149e07a77d6665611b2fad1b Mon Sep 17 00:00:00 2001
From: Ashley Wu <wangchew@andrew.cmu.edu>
Date: Mon, 25 Apr 2022 00:15:51 -0400
Subject: [PATCH 49/56] fix quote error

---
 egs2/mediaspeech/asr1/local/data.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/egs2/mediaspeech/asr1/local/data.sh b/egs2/mediaspeech/asr1/local/data.sh
index a904d143883..a8284fdeb88 100755
--- a/egs2/mediaspeech/asr1/local/data.sh
+++ b/egs2/mediaspeech/asr1/local/data.sh
@@ -35,7 +35,7 @@ log "data preparation started"
 
 if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then 
     log "stage1: Download data to ${MEDIASPEECH}"
-    log "The default data of this recipe is from mediaspeech - french 
+    log "The default data of this recipe is from mediaspeech - french"
     local/download_and_untar.sh ${MEDIASPEECH} ${data_url} FR.tgz
 fi
 

From 5dcda79ffafc2894f6545bfac46f586dfc0a1d8c Mon Sep 17 00:00:00 2001
From: Siddhant <siddhana@br012.ib.bridges2.psc.edu>
Date: Mon, 25 Apr 2022 01:33:46 -0400
Subject: [PATCH 50/56] Remove unnecessary configs

---
 egs2/slue-voxpopuli/asr1/README.md            |   2 -
 .../asr1/conf/decode_asr_low_ctc.yaml         |   6 -
 .../asr1/conf/decode_asr_no_ctc.yaml          |   6 -
 ...nformer_lr1e-3_warmup5k_wavlm_conv2d2.yaml |  88 -------------
 ...up5k_wav2vec2_conv2d2_wdecay1e-5_bert.yaml |  99 ---------------
 ...5k_wav2vec2_conv2d2_wdecay1e-5_bert_1.yaml | 118 ------------------
 ...k_wav2vec2_conv2d2_wdecay1e-5_deberta.yaml |  99 ---------------
 ...wav2vec2_conv2d2_wdecay1e-5_deberta_1.yaml | 118 ------------------
 ...nformer_lr3e-3_warmup5k_wavlm_conv2d2.yaml |  88 -------------
 ...nformer_lr5e-4_warmup5k_wavlm_conv2d2.yaml |  88 -------------
 10 files changed, 712 deletions(-)
 delete mode 100644 egs2/slue-voxpopuli/asr1/README.md
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/decode_asr_low_ctc.yaml
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/decode_asr_no_ctc.yaml
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr1e-3_warmup5k_wavlm_conv2d2.yaml
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_bert.yaml
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_bert_1.yaml
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_deberta.yaml
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_deberta_1.yaml
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr3e-3_warmup5k_wavlm_conv2d2.yaml
 delete mode 100644 egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr5e-4_warmup5k_wavlm_conv2d2.yaml

diff --git a/egs2/slue-voxpopuli/asr1/README.md b/egs2/slue-voxpopuli/asr1/README.md
deleted file mode 100644
index 139597f9cb0..00000000000
--- a/egs2/slue-voxpopuli/asr1/README.md
+++ /dev/null
@@ -1,2 +0,0 @@
-
-
diff --git a/egs2/slue-voxpopuli/asr1/conf/decode_asr_low_ctc.yaml b/egs2/slue-voxpopuli/asr1/conf/decode_asr_low_ctc.yaml
deleted file mode 100644
index 7aa1597192c..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/decode_asr_low_ctc.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-beam_size: 10
-ctc_weight: 0.1
-lm_weight: 0.0
-maxlenratio: 0.0
-minlenratio: 0.0
-penalty: 0.0
diff --git a/egs2/slue-voxpopuli/asr1/conf/decode_asr_no_ctc.yaml b/egs2/slue-voxpopuli/asr1/conf/decode_asr_no_ctc.yaml
deleted file mode 100644
index a86d524e203..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/decode_asr_no_ctc.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-beam_size: 10
-ctc_weight: 0.0
-lm_weight: 0.0
-maxlenratio: 0.0
-minlenratio: 0.0
-penalty: 0.0
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr1e-3_warmup5k_wavlm_conv2d2.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr1e-3_warmup5k_wavlm_conv2d2.yaml
deleted file mode 100644
index 842684710a2..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr1e-3_warmup5k_wavlm_conv2d2.yaml
+++ /dev/null
@@ -1,88 +0,0 @@
-specaug: specaug
-specaug_conf:
-    apply_time_warp: true
-    time_warp_window: 5
-    time_warp_mode: bicubic
-    apply_freq_mask: true
-    freq_mask_width_range:
-    - 0
-    - 27
-    num_freq_mask: 2
-    apply_time_mask: true
-    time_mask_width_ratio_range:
-    - 0.
-    - 0.05
-    num_time_mask: 2
-
-encoder: conformer
-encoder_conf:
-    output_size: 256
-    attention_heads: 4
-    linear_units: 1024
-    num_blocks: 12
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: conv2d2
-    normalize_before: true
-    macaron_style: true
-    rel_pos_type: latest
-    pos_enc_layer_type: rel_pos
-    selfattention_layer_type: rel_selfattn
-    activation_type: swish
-    use_cnn_module: true
-    cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-    attention_heads: 4
-    linear_units: 2048
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    self_attention_dropout_rate: 0.1
-    src_attention_dropout_rate: 0.1
-
-model_conf:
-    ctc_weight: 0.3
-    lsm_weight: 0.1
-    length_normalized_loss: false
-    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
-
-frontend: s3prl
-frontend_conf:
-    frontend_conf:
-        upstream: wavlm_large  # Note: If the upstream is changed, please change the input_size in the preencoder.
-    download_dir: ./hub
-    multilayer_feature: true
-
-preencoder: linear
-preencoder_conf:
-    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
-    output_size: 80
-
-freeze_param:
-    - "frontend.upstream"
-
-seed: 2022
-log_interval: 100
-num_workers: 2
-batch_type: numel
-batch_bins: 6000000
-accum_grad: 2
-max_epoch: 70
-patience: none
-init: none
-best_model_criterion:
--   - valid
-    - acc
-    - max
-keep_nbest_models: 10
-
-optim: adam
-optim_conf:
-    lr: 0.001
-    weight_decay: 0.000001
-scheduler: warmuplr
-scheduler_conf:
-    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_bert.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_bert.yaml
deleted file mode 100644
index 311b9e8a562..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_bert.yaml
+++ /dev/null
@@ -1,99 +0,0 @@
-specaug: specaug
-specaug_conf:
-    apply_time_warp: true
-    time_warp_window: 5
-    time_warp_mode: bicubic
-    apply_freq_mask: true
-    freq_mask_width_range:
-    - 0
-    - 27
-    num_freq_mask: 2
-    apply_time_mask: true
-    time_mask_width_ratio_range:
-    - 0.
-    - 0.05
-    num_time_mask: 2
-
-encoder: conformer
-encoder_conf:
-    output_size: 256
-    attention_heads: 4
-    linear_units: 1024
-    num_blocks: 12
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: conv2d2
-    normalize_before: true
-    macaron_style: true
-    rel_pos_type: latest
-    pos_enc_layer_type: rel_pos
-    selfattention_layer_type: rel_selfattn
-    activation_type: swish
-    use_cnn_module: true
-    cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-    attention_heads: 4
-    linear_units: 2048
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    self_attention_dropout_rate: 0.1
-    src_attention_dropout_rate: 0.1
-
-model_conf:
-    ctc_weight: 0.3
-    lsm_weight: 0.1
-    length_normalized_loss: false
-    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
-
-frontend: s3prl
-frontend_conf:
-    frontend_conf:
-        upstream:  wavlm_large  # Note: If the upstream is changed, please change the input_size in the preencoder.
-    download_dir: ./hub
-    multilayer_feature: true
-
-preencoder: linear
-preencoder_conf:
-    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
-    output_size: 80
-
-postdecoder: hugging_face_transformers
-postdecoder_conf:
-    # pick up a model from https://huggingface.co/models?filter=transformers
-    # most of models should work, but maybe some don't
-    # known to work: bert, gpt2, xlnet, roberta, mpnet, t5, bart
-    # xlnet currently works for single gpu only
-    model_name_or_path: "bert-base-cased"
-    output_size: 256
-
-freeze_param:
-    - "frontend.upstream"
-    - "encoder"
-    - "postdecoder.model"
-
-seed: 2022
-log_interval: 100
-num_workers: 2
-batch_type: numel
-batch_bins: 6000000
-accum_grad: 2
-max_epoch: 70
-patience: none
-init: none
-best_model_criterion:
--   - valid
-    - acc
-    - max
-keep_nbest_models: 10
-
-optim: adam
-optim_conf:
-    lr: 0.002
-    weight_decay: 0.000001
-scheduler: warmuplr
-scheduler_conf:
-    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_bert_1.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_bert_1.yaml
deleted file mode 100644
index 45f9a96b01f..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_bert_1.yaml
+++ /dev/null
@@ -1,118 +0,0 @@
-specaug: specaug
-specaug_conf:
-    apply_time_warp: true
-    time_warp_window: 5
-    time_warp_mode: bicubic
-    apply_freq_mask: true
-    freq_mask_width_range:
-    - 0
-    - 27
-    num_freq_mask: 2
-    apply_time_mask: true
-    time_mask_width_ratio_range:
-    - 0.
-    - 0.05
-    num_time_mask: 2
-
-encoder: conformer
-encoder_conf:
-    output_size: 256
-    attention_heads: 4
-    linear_units: 1024
-    num_blocks: 12
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: conv2d2
-    normalize_before: true
-    macaron_style: true
-    rel_pos_type: latest
-    pos_enc_layer_type: rel_pos
-    selfattention_layer_type: rel_selfattn
-    activation_type: swish
-    use_cnn_module: true
-    cnn_module_kernel: 31
-
-deliberationencoder: conformer
-deliberationencoder_conf:
-    output_size: 256
-    attention_heads: 4
-    linear_units: 1024
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: linear
-    normalize_before: true
-    macaron_style: true
-    rel_pos_type: latest
-    pos_enc_layer_type: rel_pos
-    selfattention_layer_type: rel_selfattn
-    activation_type: swish
-    use_cnn_module: true
-    cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-    attention_heads: 4
-    linear_units: 2048
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    self_attention_dropout_rate: 0.1
-    src_attention_dropout_rate: 0.1
-
-model_conf:
-    ctc_weight: 0.3
-    lsm_weight: 0.1
-    length_normalized_loss: false
-    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
-
-frontend: s3prl
-frontend_conf:
-    frontend_conf:
-        upstream:  wavlm_large  # Note: If the upstream is changed, please change the input_size in the preencoder.
-    download_dir: ./hub
-    multilayer_feature: true
-
-preencoder: linear
-preencoder_conf:
-    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
-    output_size: 80
-
-postdecoder: hugging_face_transformers
-postdecoder_conf:
-    # pick up a model from https://huggingface.co/models?filter=transformers
-    # most of models should work, but maybe some don't
-    # known to work: bert, gpt2, xlnet, roberta, mpnet, t5, bart
-    # xlnet currently works for single gpu only
-    model_name_or_path: "bert-base-cased"
-    output_size: 256
-
-freeze_param:
-    - "frontend.upstream"
-    - "encoder"
-    - "postdecoder.model"
-
-seed: 2022
-log_interval: 100
-num_workers: 2
-batch_type: numel
-batch_bins: 6000000
-accum_grad: 2
-max_epoch: 65
-patience: none
-init: none
-best_model_criterion:
--   - valid
-    - acc
-    - max
-keep_nbest_models: 10
-
-optim: adam
-optim_conf:
-    lr: 0.002
-    weight_decay: 0.000001
-scheduler: warmuplr
-scheduler_conf:
-    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_deberta.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_deberta.yaml
deleted file mode 100644
index 081b08c1ea6..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_deberta.yaml
+++ /dev/null
@@ -1,99 +0,0 @@
-specaug: specaug
-specaug_conf:
-    apply_time_warp: true
-    time_warp_window: 5
-    time_warp_mode: bicubic
-    apply_freq_mask: true
-    freq_mask_width_range:
-    - 0
-    - 27
-    num_freq_mask: 2
-    apply_time_mask: true
-    time_mask_width_ratio_range:
-    - 0.
-    - 0.05
-    num_time_mask: 2
-
-encoder: conformer
-encoder_conf:
-    output_size: 256
-    attention_heads: 4
-    linear_units: 1024
-    num_blocks: 12
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: conv2d2
-    normalize_before: true
-    macaron_style: true
-    rel_pos_type: latest
-    pos_enc_layer_type: rel_pos
-    selfattention_layer_type: rel_selfattn
-    activation_type: swish
-    use_cnn_module: true
-    cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-    attention_heads: 4
-    linear_units: 2048
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    self_attention_dropout_rate: 0.1
-    src_attention_dropout_rate: 0.1
-
-model_conf:
-    ctc_weight: 0.3
-    lsm_weight: 0.1
-    length_normalized_loss: false
-    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
-
-frontend: s3prl
-frontend_conf:
-    frontend_conf:
-        upstream:  wavlm_large  # Note: If the upstream is changed, please change the input_size in the preencoder.
-    download_dir: ./hub
-    multilayer_feature: true
-
-preencoder: linear
-preencoder_conf:
-    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
-    output_size: 80
-
-postdecoder: hugging_face_transformers
-postdecoder_conf:
-    # pick up a model from https://huggingface.co/models?filter=transformers
-    # most of models should work, but maybe some don't
-    # known to work: bert, gpt2, xlnet, roberta, mpnet, t5, bart
-    # xlnet currently works for single gpu only
-    model_name_or_path: "microsoft/deberta-large"
-    output_size: 256
-
-freeze_param:
-    - "frontend.upstream"
-    - "encoder"
-    - "postdecoder.model"
-
-seed: 2022
-log_interval: 100
-num_workers: 2
-batch_type: numel
-batch_bins: 6000000
-accum_grad: 2
-max_epoch: 60
-patience: none
-init: none
-best_model_criterion:
--   - valid
-    - acc
-    - max
-keep_nbest_models: 10
-
-optim: adam
-optim_conf:
-    lr: 0.002
-    weight_decay: 0.000001
-scheduler: warmuplr
-scheduler_conf:
-    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_deberta_1.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_deberta_1.yaml
deleted file mode 100644
index 7bd8c40432b..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr2e-3_warmup5k_wav2vec2_conv2d2_wdecay1e-5_deberta_1.yaml
+++ /dev/null
@@ -1,118 +0,0 @@
-specaug: specaug
-specaug_conf:
-    apply_time_warp: true
-    time_warp_window: 5
-    time_warp_mode: bicubic
-    apply_freq_mask: true
-    freq_mask_width_range:
-    - 0
-    - 27
-    num_freq_mask: 2
-    apply_time_mask: true
-    time_mask_width_ratio_range:
-    - 0.
-    - 0.05
-    num_time_mask: 2
-
-encoder: conformer
-encoder_conf:
-    output_size: 256
-    attention_heads: 4
-    linear_units: 1024
-    num_blocks: 12
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: conv2d2
-    normalize_before: true
-    macaron_style: true
-    rel_pos_type: latest
-    pos_enc_layer_type: rel_pos
-    selfattention_layer_type: rel_selfattn
-    activation_type: swish
-    use_cnn_module: true
-    cnn_module_kernel: 31
-
-deliberationencoder: conformer
-deliberationencoder_conf:
-    output_size: 256
-    attention_heads: 4
-    linear_units: 1024
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: linear
-    normalize_before: true
-    macaron_style: true
-    rel_pos_type: latest
-    pos_enc_layer_type: rel_pos
-    selfattention_layer_type: rel_selfattn
-    activation_type: swish
-    use_cnn_module: true
-    cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-    attention_heads: 4
-    linear_units: 2048
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    self_attention_dropout_rate: 0.1
-    src_attention_dropout_rate: 0.1
-
-model_conf:
-    ctc_weight: 0.3
-    lsm_weight: 0.1
-    length_normalized_loss: false
-    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
-
-frontend: s3prl
-frontend_conf:
-    frontend_conf:
-        upstream:  wavlm_large  # Note: If the upstream is changed, please change the input_size in the preencoder.
-    download_dir: ./hub
-    multilayer_feature: true
-
-preencoder: linear
-preencoder_conf:
-    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
-    output_size: 80
-
-postdecoder: hugging_face_transformers
-postdecoder_conf:
-    # pick up a model from https://huggingface.co/models?filter=transformers
-    # most of models should work, but maybe some don't
-    # known to work: bert, gpt2, xlnet, roberta, mpnet, t5, bart
-    # xlnet currently works for single gpu only
-    model_name_or_path: "microsoft/deberta-large"
-    output_size: 256
-
-freeze_param:
-    - "frontend.upstream"
-    - "encoder"
-    - "postdecoder.model"
-
-seed: 2022
-log_interval: 100
-num_workers: 2
-batch_type: numel
-batch_bins: 6000000
-accum_grad: 2
-max_epoch: 55
-patience: none
-init: none
-best_model_criterion:
--   - valid
-    - acc
-    - max
-keep_nbest_models: 10
-
-optim: adam
-optim_conf:
-    lr: 0.002
-    weight_decay: 0.000001
-scheduler: warmuplr
-scheduler_conf:
-    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr3e-3_warmup5k_wavlm_conv2d2.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr3e-3_warmup5k_wavlm_conv2d2.yaml
deleted file mode 100644
index 5d5852133cc..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr3e-3_warmup5k_wavlm_conv2d2.yaml
+++ /dev/null
@@ -1,88 +0,0 @@
-specaug: specaug
-specaug_conf:
-    apply_time_warp: true
-    time_warp_window: 5
-    time_warp_mode: bicubic
-    apply_freq_mask: true
-    freq_mask_width_range:
-    - 0
-    - 27
-    num_freq_mask: 2
-    apply_time_mask: true
-    time_mask_width_ratio_range:
-    - 0.
-    - 0.05
-    num_time_mask: 2
-
-encoder: conformer
-encoder_conf:
-    output_size: 256
-    attention_heads: 4
-    linear_units: 1024
-    num_blocks: 12
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: conv2d2
-    normalize_before: true
-    macaron_style: true
-    rel_pos_type: latest
-    pos_enc_layer_type: rel_pos
-    selfattention_layer_type: rel_selfattn
-    activation_type: swish
-    use_cnn_module: true
-    cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-    attention_heads: 4
-    linear_units: 2048
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    self_attention_dropout_rate: 0.1
-    src_attention_dropout_rate: 0.1
-
-model_conf:
-    ctc_weight: 0.3
-    lsm_weight: 0.1
-    length_normalized_loss: false
-    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
-
-frontend: s3prl
-frontend_conf:
-    frontend_conf:
-        upstream: wavlm_large  # Note: If the upstream is changed, please change the input_size in the preencoder.
-    download_dir: ./hub
-    multilayer_feature: true
-
-preencoder: linear
-preencoder_conf:
-    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
-    output_size: 80
-
-freeze_param:
-    - "frontend.upstream"
-
-seed: 2022
-log_interval: 100
-num_workers: 2
-batch_type: numel
-batch_bins: 6000000
-accum_grad: 2
-max_epoch: 70
-patience: none
-init: none
-best_model_criterion:
--   - valid
-    - acc
-    - max
-keep_nbest_models: 10
-
-optim: adam
-optim_conf:
-    lr: 0.003
-    weight_decay: 0.000001
-scheduler: warmuplr
-scheduler_conf:
-    warmup_steps: 5000
diff --git a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr5e-4_warmup5k_wavlm_conv2d2.yaml b/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr5e-4_warmup5k_wavlm_conv2d2.yaml
deleted file mode 100644
index 8cb1dfe8737..00000000000
--- a/egs2/slue-voxpopuli/asr1/conf/tuning_wavlm/train_asr_conformer_lr5e-4_warmup5k_wavlm_conv2d2.yaml
+++ /dev/null
@@ -1,88 +0,0 @@
-specaug: specaug
-specaug_conf:
-    apply_time_warp: true
-    time_warp_window: 5
-    time_warp_mode: bicubic
-    apply_freq_mask: true
-    freq_mask_width_range:
-    - 0
-    - 27
-    num_freq_mask: 2
-    apply_time_mask: true
-    time_mask_width_ratio_range:
-    - 0.
-    - 0.05
-    num_time_mask: 2
-
-encoder: conformer
-encoder_conf:
-    output_size: 256
-    attention_heads: 4
-    linear_units: 1024
-    num_blocks: 12
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    attention_dropout_rate: 0.1
-    input_layer: conv2d2
-    normalize_before: true
-    macaron_style: true
-    rel_pos_type: latest
-    pos_enc_layer_type: rel_pos
-    selfattention_layer_type: rel_selfattn
-    activation_type: swish
-    use_cnn_module: true
-    cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-    attention_heads: 4
-    linear_units: 2048
-    num_blocks: 6
-    dropout_rate: 0.1
-    positional_dropout_rate: 0.1
-    self_attention_dropout_rate: 0.1
-    src_attention_dropout_rate: 0.1
-
-model_conf:
-    ctc_weight: 0.3
-    lsm_weight: 0.1
-    length_normalized_loss: false
-    extract_feats_in_collect_stats: false   # Note: false means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend
-
-frontend: s3prl
-frontend_conf:
-    frontend_conf:
-        upstream: wavlm_large  # Note: If the upstream is changed, please change the input_size in the preencoder.
-    download_dir: ./hub
-    multilayer_feature: true
-
-preencoder: linear
-preencoder_conf:
-    input_size: 1024  # Note: If the upstream is changed, please change this value accordingly.
-    output_size: 80
-
-freeze_param:
-    - "frontend.upstream"
-
-seed: 2022
-log_interval: 100
-num_workers: 2
-batch_type: numel
-batch_bins: 6000000
-accum_grad: 2
-max_epoch: 70
-patience: none
-init: none
-best_model_criterion:
--   - valid
-    - acc
-    - max
-keep_nbest_models: 10
-
-optim: adam
-optim_conf:
-    lr: 0.0005
-    weight_decay: 0.000001
-scheduler: warmuplr
-scheduler_conf:
-    warmup_steps: 5000

From 5d67e5a627f4f429476af28e262165246888a418 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Mon, 25 Apr 2022 21:49:24 +0800
Subject: [PATCH 51/56] Add support for additional stats in enh loss
 calculation

---
 espnet2/enh/loss/criterions/tf_domain.py      | 56 +++++++++++++++++++
 espnet2/enh/loss/wrappers/fixed_order.py      | 11 +++-
 espnet2/enh/loss/wrappers/pit_solver.py       | 25 +++++++--
 .../enh/loss/criterions/test_tf_domain.py     | 17 ++++++
 .../enh/loss/wrappers/test_pit_solver.py      | 24 ++++++++
 5 files changed, 125 insertions(+), 8 deletions(-)

diff --git a/espnet2/enh/loss/criterions/tf_domain.py b/espnet2/enh/loss/criterions/tf_domain.py
index 0b3012bb49a..f32fdc4dfb7 100644
--- a/espnet2/enh/loss/criterions/tf_domain.py
+++ b/espnet2/enh/loss/criterions/tf_domain.py
@@ -265,3 +265,59 @@ def forward(self, ref, inf) -> torch.Tensor:
         else:
             raise ValueError("`ref` and `inf` must be complex tensors.")
         return coh_loss
+
+
+class FrequencyDomainCrossEntropy(FrequencyDomainLoss):
+    def __init__(self, compute_on_mask=False, mask_type=None):
+        super().__init__()
+        self._compute_on_mask = False
+        self._mask_type = None
+
+    @property
+    def compute_on_mask(self) -> bool:
+        return self._compute_on_mask
+
+    @property
+    def mask_type(self) -> str:
+        return self._mask_type
+
+    @property
+    def name(self) -> str:
+        if self.compute_on_mask:
+            return f"CE_on_{self.mask_type}"
+        else:
+            return "CE_on_Spec"
+
+    def forward(self, ref, inf) -> torch.Tensor:
+        """time-frequency cross-entropy loss.
+
+        Args:
+            ref: (Batch, T) or (Batch, T, C)
+            inf: (Batch, T, nclass) or (Batch, T, C, nclass)
+        Returns:
+            loss: (Batch,)
+        """
+        assert ref.shape[0] == inf.shape[0] and ref.shape[1] == inf.shape[1], (
+            ref.shape,
+            inf.shape,
+        )
+
+        if ref.dim() == 2:
+            loss = torch.nn.functional.cross_entropy(
+                inf.permute(0, 2, 1), ref, reduction="none"
+            ).mean(dim=1)
+        elif ref.dim() == 3:
+            loss = torch.nn.functional.cross_entropy(
+                inf.permute(0, 3, 1, 2), ref, reduction="none"
+            ).mean(dim=[1, 2])
+        else:
+            raise ValueError(
+                "Invalid input shape: ref={}, inf={}".format(ref.shape, inf.shape)
+            )
+
+        with torch.no_grad():
+            pred = inf.argmax(-1)
+            acc = (pred == ref).float().mean()
+            self.stats = {"acc": float(acc.cpu() * 100)}
+
+        return loss
diff --git a/espnet2/enh/loss/wrappers/fixed_order.py b/espnet2/enh/loss/wrappers/fixed_order.py
index d37bcfe80d6..c08af7d43e4 100644
--- a/espnet2/enh/loss/wrappers/fixed_order.py
+++ b/espnet2/enh/loss/wrappers/fixed_order.py
@@ -1,3 +1,5 @@
+from collections import defaultdict
+
 import torch
 
 from espnet2.enh.loss.criterions.abs_loss import AbsEnhLoss
@@ -26,11 +28,14 @@ def forward(self, ref, inf, others={}):
         num_spk = len(ref)
 
         loss = 0.0
-
+        stats = defaultdict(list)
         for r, i in zip(ref, inf):
             loss += torch.mean(self.criterion(r, i)) / num_spk
+            for k, v in getattr(self.criterion, "stats", {}).items():
+                stats[k].append(v)
 
-        stats = dict()
+        for k, v in stats.items():
+            stats[k] = sum(v) / len(v) if v else 0
         stats[self.criterion.name] = loss.detach()
 
-        return loss.mean(), stats, {}
+        return loss.mean(), dict(stats), {}
diff --git a/espnet2/enh/loss/wrappers/pit_solver.py b/espnet2/enh/loss/wrappers/pit_solver.py
index 72ea7be354b..56c78064196 100644
--- a/espnet2/enh/loss/wrappers/pit_solver.py
+++ b/espnet2/enh/loss/wrappers/pit_solver.py
@@ -1,3 +1,4 @@
+from collections import defaultdict
 from itertools import permutations
 
 import torch
@@ -43,9 +44,20 @@ def forward(self, ref, inf, others={}):
         assert len(ref) == len(inf), (len(ref), len(inf))
         num_spk = len(ref)
 
+        stats = defaultdict(list)
+
+        def pre_hook(func, *args, **kwargs):
+            ret = func(*args, **kwargs)
+            for k, v in getattr(self.criterion, "stats", {}).items():
+                stats[k].append(v)
+            return ret
+
         def pair_loss(permutation):
             return sum(
-                [self.criterion(ref[s], inf[t]) for s, t in enumerate(permutation)]
+                [
+                    pre_hook(self.criterion, ref[s], inf[t])
+                    for s, t in enumerate(permutation)
+                ]
             ) / len(permutation)
 
         if self.independent_perm or perm is None:
@@ -64,8 +76,10 @@ def pair_loss(permutation):
                 [
                     torch.tensor(
                         [
-                            self.criterion(
-                                ref[s][batch].unsqueeze(0), inf[t][batch].unsqueeze(0)
+                            pre_hook(
+                                self.criterion,
+                                ref[s][batch].unsqueeze(0),
+                                inf[t][batch].unsqueeze(0),
                             )
                             for s, t in enumerate(p)
                         ]
@@ -76,7 +90,8 @@ def pair_loss(permutation):
 
         loss = loss.mean()
 
-        stats = dict()
+        for k, v in stats.items():
+            stats[k] = sum(v) / len(v) if v else 0
         stats[self.criterion.name] = loss.detach()
 
-        return loss.mean(), stats, {"perm": perm}
+        return loss.mean(), dict(stats), {"perm": perm}
diff --git a/test/espnet2/enh/loss/criterions/test_tf_domain.py b/test/espnet2/enh/loss/criterions/test_tf_domain.py
index 1a56039b475..41999948887 100644
--- a/test/espnet2/enh/loss/criterions/test_tf_domain.py
+++ b/test/espnet2/enh/loss/criterions/test_tf_domain.py
@@ -5,6 +5,7 @@
 from torch_complex import ComplexTensor
 
 from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainAbsCoherence
+from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainCrossEntropy
 from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainL1
 from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainMSE
 
@@ -76,3 +77,19 @@ def test_tf_coh_criterion_invalid_forward(input_ch):
             criterion(ref_spec[0], inf_spec[0])
         else:
             criterion(ref_spec[0, 0], inf_spec[0, 0])
+
+
+@pytest.mark.parametrize("input_ch", [1, 2])
+def test_tf_ce_criterion_forward(input_ch):
+
+    criterion = FrequencyDomainCrossEntropy()
+
+    batch = 2
+    ncls = 200
+    shape = (batch, 10, ncls) if input_ch == 1 else (batch, 10, input_ch, ncls)
+    label_shape = (batch, 10) if input_ch == 1 else (batch, 10, input_ch)
+    inf_spec = torch.rand(*shape)
+    ref_spec = torch.randint(0, ncls, label_shape)
+
+    loss = criterion(ref_spec, inf_spec)
+    assert loss.shape == (batch,), "Invlid loss shape with " + criterion.name
diff --git a/test/espnet2/enh/loss/wrappers/test_pit_solver.py b/test/espnet2/enh/loss/wrappers/test_pit_solver.py
index 40248a6b6bd..ddba099e17e 100644
--- a/test/espnet2/enh/loss/wrappers/test_pit_solver.py
+++ b/test/espnet2/enh/loss/wrappers/test_pit_solver.py
@@ -1,6 +1,8 @@
 import pytest
 import torch
+import torch.nn.functional as F
 
+from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainCrossEntropy
 from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainL1
 from espnet2.enh.loss.wrappers.pit_solver import PITSolver
 
@@ -23,3 +25,25 @@ def test_PITSolver_forward(num_spk):
 
     solver = PITSolver(FrequencyDomainL1(), independent_perm=False)
     loss, stats, others = solver(ref, inf, {"perm": perm})
+
+
+@pytest.mark.parametrize("num_spk", [1, 2, 3])
+def test_PITSolver_tf_ce_forward(num_spk):
+
+    batch = 2
+    ncls = 100
+    ref = [torch.randint(0, ncls, (batch, 10)) for spk in range(num_spk)]
+    bias = [F.one_hot(y) for y in ref]
+    bias = [F.pad(y, (0, ncls - y.size(-1))) for y in bias]
+    inf = [torch.rand(batch, 10, ncls) + bias[spk] for spk in range(num_spk)]
+    solver = PITSolver(FrequencyDomainCrossEntropy(), independent_perm=True)
+
+    loss, stats, others = solver(ref, inf)
+    perm = others["perm"]
+    correct_perm = list(range(num_spk))
+    assert perm[0].equal(torch.tensor(correct_perm)), (perm, correct_perm)
+
+    # test for independent_perm is False
+
+    solver = PITSolver(FrequencyDomainCrossEntropy(), independent_perm=False)
+    loss, stats, others = solver(ref, inf, {"perm": perm})

From 9c6d5cbaeccdfc90020de558820c07a13a43f8fc Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Mon, 25 Apr 2022 22:27:14 +0800
Subject: [PATCH 52/56] Reformat enh.py

---
 espnet2/enh/loss/criterions/time_domain.py |  4 ++--
 espnet2/tasks/enh.py                       | 26 ++++++++++++----------
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/espnet2/enh/loss/criterions/time_domain.py b/espnet2/enh/loss/criterions/time_domain.py
index e2060079c70..62b2ef262e2 100644
--- a/espnet2/enh/loss/criterions/time_domain.py
+++ b/espnet2/enh/loss/criterions/time_domain.py
@@ -217,7 +217,7 @@ def __init__(self):
 
     @property
     def name(self) -> str:
-        return "time_MSE_loss"
+        return "TD_MSE_loss"
 
     def forward(self, ref, inf) -> torch.Tensor:
         """Time-domain MSE loss forward.
@@ -248,7 +248,7 @@ def __init__(self):
 
     @property
     def name(self) -> str:
-        return "time_L1_loss"
+        return "TD_L1_loss"
 
     def forward(self, ref, inf) -> torch.Tensor:
         """Time-domain L1 loss forward.
diff --git a/espnet2/tasks/enh.py b/espnet2/tasks/enh.py
index 57608365180..f0f3e3c03b9 100644
--- a/espnet2/tasks/enh.py
+++ b/espnet2/tasks/enh.py
@@ -65,17 +65,17 @@
 separator_choices = ClassChoices(
     name="separator",
     classes=dict(
+        asteroid=AsteroidModel_Converter,
+        conformer=ConformerSeparator,
+        dc_crn=DC_CRNSeparator,
+        dccrn=DCCRNSeparator,
+        dprnn=DPRNNSeparator,
+        fasnet=FaSNetSeparator,
         rnn=RNNSeparator,
         skim=SkiMSeparator,
         tcn=TCNSeparator,
-        dc_crn=DC_CRNSeparator,
-        dprnn=DPRNNSeparator,
-        dccrn=DCCRNSeparator,
         transformer=TransformerSeparator,
-        conformer=ConformerSeparator,
         wpe_beamformer=NeuralBeamformer,
-        asteroid=AsteroidModel_Converter,
-        fasnet=FaSNetSeparator,
     ),
     type_check=AbsSeparator,
     default="rnn",
@@ -98,15 +98,17 @@
 criterion_choices = ClassChoices(
     name="criterions",
     classes=dict(
-        snr=SNRLoss,
-        sdr=SDRLoss,
         ci_sdr=CISDRLoss,
-        si_snr=SISNRLoss,
         coh=FrequencyDomainAbsCoherence,
-        mse=FrequencyDomainMSE,
+        sdr=SDRLoss,
+        si_snr=SISNRLoss,
+        snr=SNRLoss,
         l1=FrequencyDomainL1,
-        mse_t=TimeDomainMSE,
-        l1_t=TimeDomainL1,
+        l1_fd=FrequencyDomainL1,
+        l1_td=TimeDomainL1,
+        mse=FrequencyDomainMSE,
+        mse_fd=FrequencyDomainMSE,
+        mse_td=TimeDomainMSE,
     ),
     type_check=AbsEnhLoss,
     default=None,

From 19bee5f8fb7197e2b90fef284901f2dd96eb9a0f Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Mon, 25 Apr 2022 23:31:57 +0800
Subject: [PATCH 53/56] Allow specifying the criterion name by users

---
 espnet2/enh/loss/criterions/tf_domain.py   | 48 ++++++++++++++--------
 espnet2/enh/loss/criterions/time_domain.py | 33 ++++++++++-----
 2 files changed, 53 insertions(+), 28 deletions(-)

diff --git a/espnet2/enh/loss/criterions/tf_domain.py b/espnet2/enh/loss/criterions/tf_domain.py
index f32fdc4dfb7..3edd78b3684 100644
--- a/espnet2/enh/loss/criterions/tf_domain.py
+++ b/espnet2/enh/loss/criterions/tf_domain.py
@@ -117,11 +117,18 @@ def create_mask_label(self, mix_spec, ref_spec, noise_spec=None):
 
 
 class FrequencyDomainMSE(FrequencyDomainLoss):
-    def __init__(self, compute_on_mask=False, mask_type="IBM"):
+    def __init__(self, compute_on_mask=False, mask_type="IBM", name=None):
         super().__init__()
         self._compute_on_mask = compute_on_mask
         self._mask_type = mask_type
 
+        if name is not None:
+            self._name = name
+        elif self.compute_on_mask:
+            self._name = f"MSE_on_{self.mask_type}"
+        else:
+            self._name = "MSE_on_Spec"
+
     @property
     def compute_on_mask(self) -> bool:
         return self._compute_on_mask
@@ -132,10 +139,7 @@ def mask_type(self) -> str:
 
     @property
     def name(self) -> str:
-        if self.compute_on_mask:
-            return f"MSE_on_{self.mask_type}"
-        else:
-            return "MSE_on_Spec"
+        return self._name
 
     def forward(self, ref, inf) -> torch.Tensor:
         """time-frequency MSE loss.
@@ -165,11 +169,18 @@ def forward(self, ref, inf) -> torch.Tensor:
 
 
 class FrequencyDomainL1(FrequencyDomainLoss):
-    def __init__(self, compute_on_mask=False, mask_type="IBM"):
+    def __init__(self, compute_on_mask=False, mask_type="IBM", name=None):
         super().__init__()
         self._compute_on_mask = compute_on_mask
         self._mask_type = mask_type
 
+        if name is not None:
+            self._name = name
+        elif self.compute_on_mask:
+            self._name = f"L1_on_{self.mask_type}"
+        else:
+            self._name = "L1_on_Spec"
+
     @property
     def compute_on_mask(self) -> bool:
         return self._compute_on_mask
@@ -180,10 +191,7 @@ def mask_type(self) -> str:
 
     @property
     def name(self) -> str:
-        if self.compute_on_mask:
-            return f"L1_on_{self.mask_type}"
-        else:
-            return "L1_on_Spec"
+        return self._name
 
     def forward(self, ref, inf) -> torch.Tensor:
         """time-frequency L1 loss.
@@ -216,11 +224,13 @@ def forward(self, ref, inf) -> torch.Tensor:
 
 
 class FrequencyDomainAbsCoherence(FrequencyDomainLoss):
-    def __init__(self, compute_on_mask=False, mask_type=None):
+    def __init__(self, compute_on_mask=False, mask_type=None, name=None):
         super().__init__()
         self._compute_on_mask = False
         self._mask_type = None
 
+        self._name = "Coherence_on_Spec" if name is None else name
+
     @property
     def compute_on_mask(self) -> bool:
         return self._compute_on_mask
@@ -231,7 +241,7 @@ def mask_type(self) -> str:
 
     @property
     def name(self) -> str:
-        return "Coherence_on_Spec"
+        return self._name
 
     def forward(self, ref, inf) -> torch.Tensor:
         """time-frequency absolute coherence loss.
@@ -268,11 +278,18 @@ def forward(self, ref, inf) -> torch.Tensor:
 
 
 class FrequencyDomainCrossEntropy(FrequencyDomainLoss):
-    def __init__(self, compute_on_mask=False, mask_type=None):
+    def __init__(self, compute_on_mask=False, mask_type=None, name=None):
         super().__init__()
         self._compute_on_mask = False
         self._mask_type = None
 
+        if name is not None:
+            self._name = name
+        elif self.compute_on_mask:
+            self._name = f"CE_on_{self.mask_type}"
+        else:
+            self._name = "CE_on_Spec"
+
     @property
     def compute_on_mask(self) -> bool:
         return self._compute_on_mask
@@ -283,10 +300,7 @@ def mask_type(self) -> str:
 
     @property
     def name(self) -> str:
-        if self.compute_on_mask:
-            return f"CE_on_{self.mask_type}"
-        else:
-            return "CE_on_Spec"
+        return self._name
 
     def forward(self, ref, inf) -> torch.Tensor:
         """time-frequency cross-entropy loss.
diff --git a/espnet2/enh/loss/criterions/time_domain.py b/espnet2/enh/loss/criterions/time_domain.py
index 62b2ef262e2..d000b83fbbb 100644
--- a/espnet2/enh/loss/criterions/time_domain.py
+++ b/espnet2/enh/loss/criterions/time_domain.py
@@ -35,13 +35,15 @@ class CISDRLoss(TimeDomainLoss):
         loss: (Batch,)
     """
 
-    def __init__(self, filter_length=512):
+    def __init__(self, filter_length=512, name=None):
         super().__init__()
         self.filter_length = filter_length
 
+        self._name = "ci_sdr_loss" if name is None else name
+
     @property
     def name(self) -> str:
-        return "ci_sdr_loss"
+        return self._name
 
     def forward(
         self,
@@ -57,13 +59,15 @@ def forward(
 
 
 class SNRLoss(TimeDomainLoss):
-    def __init__(self, eps=EPS):
+    def __init__(self, eps=EPS, name=None):
         super().__init__()
         self.eps = float(eps)
 
+        self._name = "snr_loss" if name is None else name
+
     @property
     def name(self) -> str:
-        return "snr_loss"
+        return self._name
 
     def forward(
         self,
@@ -111,6 +115,7 @@ def __init__(
         clamp_db=None,
         zero_mean=True,
         load_diag=None,
+        name=None,
     ):
         super().__init__()
 
@@ -120,9 +125,11 @@ def __init__(
         self.zero_mean = zero_mean
         self.load_diag = load_diag
 
+        self._name = "sdr_loss" if name is None else name
+
     @property
     def name(self) -> str:
-        return "sdr_loss"
+        return self._name
 
     def forward(
         self,
@@ -170,16 +177,18 @@ class SISNRLoss(TimeDomainLoss):
             Deprecated. Keeped for compatibility.
     """
 
-    def __init__(self, clamp_db=None, zero_mean=True, eps=None):
+    def __init__(self, clamp_db=None, zero_mean=True, eps=None, name=None):
         super().__init__()
         self.clamp_db = clamp_db
         self.zero_mean = zero_mean
         if eps is not None:
             logging.warning("Eps is deprecated in si_snr loss, set clamp_db instead.")
 
+        self._name = "si_snr_loss" if name is None else name
+
     @property
     def name(self) -> str:
-        return "si_snr_loss"
+        return self._name
 
     def forward(
         self,
@@ -212,12 +221,13 @@ def forward(
 
 
 class TimeDomainMSE(TimeDomainLoss):
-    def __init__(self):
+    def __init__(self, name=None):
         super().__init__()
+        self._name = "TD_MSE_loss" if name is None else name
 
     @property
     def name(self) -> str:
-        return "TD_MSE_loss"
+        return self._name
 
     def forward(self, ref, inf) -> torch.Tensor:
         """Time-domain MSE loss forward.
@@ -243,12 +253,13 @@ def forward(self, ref, inf) -> torch.Tensor:
 
 
 class TimeDomainL1(TimeDomainLoss):
-    def __init__(self):
+    def __init__(self, name=None):
         super().__init__()
+        self._name = "TD_L1_loss" if name is None else name
 
     @property
     def name(self) -> str:
-        return "TD_L1_loss"
+        return self._name
 
     def forward(self, ref, inf) -> torch.Tensor:
         """Time-domain L1 loss forward.

From 4e6a101627def890b44fae919a534f1b3e6ceb90 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Tue, 26 Apr 2022 00:18:26 +0800
Subject: [PATCH 54/56] Fix a bug in stats aggregation when PITSolver is used

---
 espnet2/enh/loss/criterions/tf_domain.py |  8 ++++++--
 espnet2/enh/loss/wrappers/fixed_order.py |  2 +-
 espnet2/enh/loss/wrappers/pit_solver.py  | 18 +++++++++++++++---
 3 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/espnet2/enh/loss/criterions/tf_domain.py b/espnet2/enh/loss/criterions/tf_domain.py
index 3edd78b3684..1d2481e15b8 100644
--- a/espnet2/enh/loss/criterions/tf_domain.py
+++ b/espnet2/enh/loss/criterions/tf_domain.py
@@ -331,7 +331,11 @@ def forward(self, ref, inf) -> torch.Tensor:
 
         with torch.no_grad():
             pred = inf.argmax(-1)
-            acc = (pred == ref).float().mean()
-            self.stats = {"acc": float(acc.cpu() * 100)}
+            acc = (pred == ref).float()
+            if ref.dim() == 2:
+                acc = acc.mean(dim=1)
+            elif ref.dim() == 3:
+                acc = acc.mean(dim=[1, 2])
+            self.stats = {"acc": acc.cpu() * 100}
 
         return loss
diff --git a/espnet2/enh/loss/wrappers/fixed_order.py b/espnet2/enh/loss/wrappers/fixed_order.py
index c08af7d43e4..2bf1bc0f079 100644
--- a/espnet2/enh/loss/wrappers/fixed_order.py
+++ b/espnet2/enh/loss/wrappers/fixed_order.py
@@ -35,7 +35,7 @@ def forward(self, ref, inf, others={}):
                 stats[k].append(v)
 
         for k, v in stats.items():
-            stats[k] = sum(v) / len(v) if v else 0
+            stats[k] = torch.stack(v, dim=1).mean()
         stats[self.criterion.name] = loss.detach()
 
         return loss.mean(), dict(stats), {}
diff --git a/espnet2/enh/loss/wrappers/pit_solver.py b/espnet2/enh/loss/wrappers/pit_solver.py
index 56c78064196..9956f0d9edf 100644
--- a/espnet2/enh/loss/wrappers/pit_solver.py
+++ b/espnet2/enh/loss/wrappers/pit_solver.py
@@ -65,12 +65,24 @@ def pair_loss(permutation):
             device = ref[0].device
             all_permutations = list(permutations(range(num_spk)))
             losses = torch.stack([pair_loss(p) for p in all_permutations], dim=1)
-            loss, perm = torch.min(losses, dim=1)
+            loss, perm_ = torch.min(losses, dim=1)
             perm = torch.index_select(
                 torch.tensor(all_permutations, device=device, dtype=torch.long),
                 0,
-                perm,
+                perm_,
             )
+            # remove stats from unused permutations
+            for k, v in stats.items():
+                # (B, len(all_permutations), ...)
+                new_v = torch.stack(v, dim=1)
+                if new_v.dim() > 2:
+                    shapes = [1 for _ in range(new_v.dim() - 2)]
+                    perm0 = perm_.view(perm_.shape[0], 1, *shapes).expand(
+                        -1, -1, *new_v.shape[2:]
+                    )
+                else:
+                    perm0 = perm_.unsqueeze(1)
+                stats[k] = new_v.gather(1, perm0).unbind(1)
         else:
             loss = torch.tensor(
                 [
@@ -91,7 +103,7 @@ def pair_loss(permutation):
         loss = loss.mean()
 
         for k, v in stats.items():
-            stats[k] = sum(v) / len(v) if v else 0
+            stats[k] = torch.stack(v, dim=1).mean()
         stats[self.criterion.name] = loss.detach()
 
         return loss.mean(), dict(stats), {"perm": perm}

From f48c5da76be728efbbb27aefe0fe131dd86c8e3c Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Tue, 26 Apr 2022 00:36:31 +0800
Subject: [PATCH 55/56] Fix device mismatch issue when collecting stats in
 PITSolver

---
 espnet2/enh/loss/wrappers/pit_solver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/espnet2/enh/loss/wrappers/pit_solver.py b/espnet2/enh/loss/wrappers/pit_solver.py
index 9956f0d9edf..9cb810f5c9b 100644
--- a/espnet2/enh/loss/wrappers/pit_solver.py
+++ b/espnet2/enh/loss/wrappers/pit_solver.py
@@ -82,7 +82,7 @@ def pair_loss(permutation):
                     )
                 else:
                     perm0 = perm_.unsqueeze(1)
-                stats[k] = new_v.gather(1, perm0).unbind(1)
+                stats[k] = new_v.gather(1, perm0.to(device=new_v.device)).unbind(1)
         else:
             loss = torch.tensor(
                 [

From 4095f0967bff877999371dd05de18433be2682b8 Mon Sep 17 00:00:00 2001
From: Wangyou Zhang <C0me_On@163.com>
Date: Tue, 26 Apr 2022 13:43:06 +0800
Subject: [PATCH 56/56] Fix FileNotFound error when using globs

---
 egs2/TEMPLATE/enh_asr1/enh_asr.sh | 3 +++
 egs2/TEMPLATE/enh_st1/enh_st.sh   | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/egs2/TEMPLATE/enh_asr1/enh_asr.sh b/egs2/TEMPLATE/enh_asr1/enh_asr.sh
index 7ca4f9ddc77..386848539d5 100755
--- a/egs2/TEMPLATE/enh_asr1/enh_asr.sh
+++ b/egs2/TEMPLATE/enh_asr1/enh_asr.sh
@@ -529,6 +529,9 @@ if ! "${skip_data_prep}"; then
                 for extra_file in ${utt_extra_files}; do
                     # with regex to suuport multi-references
                     for single_file in "data/${dset}/${extra_file}"*; do
+                        if [ ! -f "${single_file}" ]; then
+                            continue
+                        fi
                         cp ${single_file} "${data_feats}${_suf}/${dset}"
                         expand_utt_extra_files="${expand_utt_extra_files} $(basename ${single_file})"
                     done
diff --git a/egs2/TEMPLATE/enh_st1/enh_st.sh b/egs2/TEMPLATE/enh_st1/enh_st.sh
index b6fd09c67cb..6bf520360a8 100755
--- a/egs2/TEMPLATE/enh_st1/enh_st.sh
+++ b/egs2/TEMPLATE/enh_st1/enh_st.sh
@@ -588,6 +588,9 @@ if ! "${skip_data_prep}"; then
                 for extra_file in ${utt_extra_files}; do
                     # with regex to suuport multi-references
                     for single_file in "data/${dset}/${extra_file}"*; do
+                        if [ ! -f "${single_file}" ]; then
+                            continue
+                        fi
                         cp ${single_file} "${data_feats}${_suf}/${dset}"
                         expand_utt_extra_files="${expand_utt_extra_files} $(basename ${single_file})"
                     done