Merge branch 'master' into enh_s2t

CONTRIBUTING.md

@@ -151,6 +151,11 @@ we recommend using small model parameters and avoiding dynamic imports, file acc
 more running time, you can annotate your test with `@pytest.mark.execution_timeout(sec)`.
 - For test initialization (parameters, modules, etc), you can use pytest fixtures. Refer to  [pytest fixtures](https://docs.pytest.org/en/latest/fixture.html#using-fixtures-from-classes-modules-or-projects) for more information.
 
+In addition, please follow the [PEP 8 convention](https://peps.python.org/pep-0008/) for the coding style and [Google's convention for docstrings](https://google.github.io/styleguide/pyguide.html#383-functions-and-methods).
+Below are some specific points that should be taken care of in particular:
+- [import ordering](https://peps.python.org/pep-0008/#imports)
+- Avoid writing python2-style code. For example, `super().__init__()` is preferred over `super(CLASS_NAME, self).__init()__`.
+
 
 ### 4.2 Bash scripts
 

egs/README.md

@@ -8,6 +8,7 @@ See: https://espnet.github.io/espnet/tutorial.html
 | Directory name          | Corpus name                                                  | Task                                       | Language       | URL                                                          | Note                          |
 | ----------------------- | ------------------------------------------------------------ | ------------------------------------------ | -------------- | ------------------------------------------------------------ | ----------------------------- |
 ||||
+| aesrc2020               | Accented English Speech Recognition Challenge 2020           | ASR                                        | EN             | https://arxiv.org/abs/2102.10233                                   |                               |
 | aidatatang_200zh        | Aidatatang_200zh A free Chinese Mandarin speech corpus       | ASR                                        | ZH             | http://www.openslr.org/62/                                   |                               |
 | aishell                 | AISHELL-ASR0009-OS1 Open Source Mandarin Speech Corpus       | ASR                                        | ZH             | http://www.aishelltech.com/kysjcp                            |                               |
 | aishell2                | AISHELL-2 Open Source Mandarin Speech Corpus                 | ASR                                        | ZH             | http://www.aishelltech.com/aishell_2                                                         |
@@ -49,7 +50,8 @@ See: https://espnet.github.io/espnet/tutorial.html
 | librispeech             | LibriSpeech ASR corpus                                       | ASR                                        | EN             | http://www.openslr.org/12                                    |                               |
 | libritts                | LibriTTS: A Corpus Derived from LibriSpeech for Text-to-Speech | TTS                                      | EN             | http://www.openslr.org/60/                                   |                               |
 | ljspeech                | The LJ Speech Dataset                                        | TTS                                        | EN             | https://keithito.com/LJ-Speech-Dataset/                      |                               |
-| lrs                     | The Lip Reading Sentences Dataset                            | ASR/AVSR                                       | EN             | https://www.robots.ox.ac.uk/~vgg/data/lip_reading/lrs2.html                      |                               |
+| lrs2                     | The Lip Reading Sentences 2 Dataset                            | ASR                                       | ENG             | https://www.robots.ox.ac.uk/~vgg/data/lip_reading/lrs2.html                      |                               |
+| lrs                     | The Lip Reading Sentences 2 and 3 Dataset                            | AVSR                                       | ENG             | https://www.robots.ox.ac.uk/~vgg/data/lip_reading/lrs2.html  https://www.robots.ox.ac.uk/~vgg/data/lip_reading/lrs3.html                     |                               |
 | m_ailabs                | The M-AILABS Speech Dataset                                  | TTS                                        | ~5 languages   | https://www.caito.de/2019/01/the-m-ailabs-speech-dataset/    |
 | mucs_2021               | MUCS 2021: MUltilingual and Code-Switching ASR Challenges for Low Resource Indian Languages   | ASR/Code Switching          | HI, MR, OR, TA, TE, GU, HI-EN, BN-EN | https://navana-tech.github.io/MUCS2021/data.html                    |                               |
 | mtedx                   | Multilingual TEDx | ASR/Machine Translation/Speech Translation | 13 Language pairs | http://www.openslr.org/100/                         |

egs/aesrc2020/asr1/conf/decode.yaml

@@ -0,0 +1 @@
+tuning/decode_pytorch_transformer.yaml

egs/aesrc2020/asr1/conf/lm.yaml

@@ -0,0 +1,8 @@
+# rnnlm related
+layer: 2
+unit: 650
+opt: sgd        # or adam
+batchsize: 64   # batch size in LM training
+epoch: 20      # if the data size is large, we can reduce this
+patience: 3
+maxlen: 100     # if sentence length > lm_maxlen, lm_batchsize is automatically reduced

egs/aesrc2020/asr1/conf/specaug.yaml

@@ -0,0 +1,16 @@
+process:
+  # these three processes are a.k.a. SpecAugument
+  - type: "time_warp"
+    max_time_warp: 5
+    inplace: true
+    mode: "PIL"
+  - type: "freq_mask"
+    F: 30
+    n_mask: 2
+    inplace: true
+    replace_with_zero: false
+  - type: "time_mask"
+    T: 40
+    n_mask: 2
+    inplace: true
+    replace_with_zero: false

egs/aesrc2020/asr1/conf/train.yaml

@@ -0,0 +1 @@
+tuning/train_pytorch_conformer_kernel15.yaml

egs/aesrc2020/asr1/conf/tuning/decode_pytorch_transformer.yaml

@@ -0,0 +1,8 @@
+batchsize: 0
+beam-size: 10
+penalty: 0.0
+maxlenratio: 0.0
+minlenratio: 0.0
+ctc-weight: 0.5
+lm-weight: 0.3
+ngram-weight: 0.3

egs/aesrc2020/asr1/conf/tuning/decode_rnn.yaml

@@ -0,0 +1,6 @@
+beam-size: 20
+penalty: 0.0
+maxlenratio: 0.0
+minlenratio: 0.0
+ctc-weight: 0.6
+lm-weight: 0.3

egs/aesrc2020/asr1/conf/tuning/train_pytorch_conformer_kernel15.yaml

@@ -0,0 +1,47 @@
+# network architecture
+# encoder related
+elayers: 12
+eunits: 2048
+# decoder related
+dlayers: 6
+dunits: 2048
+# attention related
+adim: 256
+aheads: 4
+
+# hybrid CTC/attention
+mtlalpha: 0.3
+
+# label smoothing
+lsm-weight: 0.1
+
+# minibatch related
+batch-size: 32
+maxlen-in: 512  # if input length  > maxlen-in, batchsize is automatically reduced
+maxlen-out: 150 # if output length > maxlen-out, batchsize is automatically reduced
+
+# optimization related
+sortagrad: 0 # Feed samples from shortest to longest ; -1: enabled for all epochs, 0: disabled, other: enabled for 'other' epochs
+opt: noam
+accum-grad: 2
+grad-clip: 5
+patience: 0
+epochs: 50
+dropout-rate: 0.1
+
+# transformer specific setting
+backend: pytorch
+model-module: "espnet.nets.pytorch_backend.e2e_asr_conformer:E2E"
+transformer-input-layer: conv2d     # encoder architecture type
+transformer-lr: 1.0
+transformer-warmup-steps: 25000
+transformer-attn-dropout-rate: 0.0
+transformer-length-normalized-loss: false
+transformer-init: pytorch
+
+# conformer specific setting
+transformer-encoder-pos-enc-layer-type: rel_pos
+transformer-encoder-selfattn-layer-type: rel_selfattn
+macaron-style: true
+use-cnn-module: true
+cnn-module-kernel: 15

egs/aesrc2020/asr1/conf/tuning/train_pytorch_conformer_kernel31.yaml

@@ -0,0 +1,47 @@
+# network architecture
+# encoder related
+elayers: 12
+eunits: 2048
+# decoder related
+dlayers: 6
+dunits: 2048
+# attention related
+adim: 256
+aheads: 4
+
+# hybrid CTC/attention
+mtlalpha: 0.3
+
+# label smoothing
+lsm-weight: 0.1
+
+# minibatch related
+batch-size: 32
+maxlen-in: 512  # if input length  > maxlen-in, batchsize is automatically reduced
+maxlen-out: 150 # if output length > maxlen-out, batchsize is automatically reduced
+
+# optimization related
+sortagrad: 0 # Feed samples from shortest to longest ; -1: enabled for all epochs, 0: disabled, other: enabled for 'other' epochs
+opt: noam
+accum-grad: 2
+grad-clip: 5
+patience: 0
+epochs: 50
+dropout-rate: 0.1
+
+# transformer specific setting
+backend: pytorch
+model-module: "espnet.nets.pytorch_backend.e2e_asr_conformer:E2E"
+transformer-input-layer: conv2d     # encoder architecture type
+transformer-lr: 1.0
+transformer-warmup-steps: 25000
+transformer-attn-dropout-rate: 0.0
+transformer-length-normalized-loss: false
+transformer-init: pytorch
+
+# conformer specific setting
+transformer-encoder-pos-enc-layer-type: rel_pos
+transformer-encoder-selfattn-layer-type: rel_selfattn
+macaron-style: true
+use-cnn-module: true
+cnn-module-kernel: 31

egs/aesrc2020/asr1/conf/tuning/train_pytorch_transformer.yaml

@@ -0,0 +1,40 @@
+# network architecture
+# encoder related
+elayers: 12
+eunits: 2048
+# decoder related
+dlayers: 6
+dunits: 2048
+# attention related
+adim: 256
+aheads: 4
+
+# hybrid CTC/attention
+mtlalpha: 0.3
+
+# label smoothing
+lsm-weight: 0.1
+
+# minibatch related
+batch-size: 32
+maxlen-in: 512  # if input length  > maxlen-in, batchsize is automatically reduced
+maxlen-out: 150 # if output length > maxlen-out, batchsize is automatically reduced
+
+# optimization related
+sortagrad: 0 # Feed samples from shortest to longest ; -1: enabled for all epochs, 0: disabled, other: enabled for 'other' epochs
+opt: noam
+accum-grad: 2
+grad-clip: 5
+patience: 0
+epochs: 50
+dropout-rate: 0.1
+
+# transformer specific setting
+backend: pytorch
+model-module: "espnet.nets.pytorch_backend.e2e_asr_transformer:E2E"
+transformer-input-layer: conv2d     # encoder architecture type
+transformer-lr: 2.0
+transformer-warmup-steps: 25000
+transformer-attn-dropout-rate: 0.0
+transformer-length-normalized-loss: false
+transformer-init: pytorch

egs/aesrc2020/asr1/conf/tuning/train_rnn.yaml

@@ -0,0 +1,31 @@
+# network architecture
+# encoder related
+etype: vggblstm     # encoder architecture type
+elayers: 3
+eunits: 1024
+eprojs: 1024
+subsample: "1_2_2_1_1" # skip every n frame from input to nth layers
+# decoder related
+dlayers: 2
+dunits: 1024
+# attention related
+atype: location
+adim: 1024
+aconv-chans: 10
+aconv-filts: 100
+
+# hybrid CTC/attention
+mtlalpha: 0.5
+
+# minibatch related
+batch-size: 30
+maxlen-in: 800  # if input length  > maxlen_in, batchsize is automatically reduced
+maxlen-out: 150 # if output length > maxlen_out, batchsize is automatically reduced
+
+# optimization related
+opt: adadelta
+epochs: 10
+patience: 0
+
+# scheduled sampling option
+sampling-probability: 0.0

egs/aesrc2020/asr1/local/create_subsets.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+. ./path.sh || exit 1;
+. ./cmd.sh || exit 1;
+
+data=$1     # data transformed into kaldi format
+
+ # divide development set for cross validation
+ if [ -d ${data} ];then
+     for i in US UK IND CHN JPN PT RU KR CA ES;do
+         ./utils/subset_data_dir.sh --spk-list local/files/cvlist/${i}_cv_spk $data/data_all $data/cv/$i
+         cat $data/cv/$i/feats.scp >> $data/cv.scp
+     done
+     ./utils/filter_scp.pl --exclude $data/cv.scp $data/data_all/feats.scp > $data/train_and_dev.scp
+     #95-5 split for dev set
+     sed -n '0~20p' $data/train_and_dev.scp > $data/dev.scp
+     ./utils/filter_scp.pl --exclude $data/dev.scp $data/train_and_dev.scp > $data/train.scp
+     ./utils/subset_data_dir.sh --utt-list $data/train.scp $data/data_all $data/train_org
+     ./utils/subset_data_dir.sh --utt-list $data/dev.scp $data/data_all $data/dev_org
+     ./utils/subset_data_dir.sh --utt-list $data/cv.scp $data/data_all $data/cv_all
+ fi
+
+echo "local/subset_data.sh succeeded"
+exit 0;

egs/aesrc2020/asr1/local/data_prep.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Copyright 2020 Audio, Speech and Language Processing Group @ NWPU (Author: Xian Shi)
+# Apache 2.0
+
+. ./path.sh || exit 1;
+. ./cmd.sh || exit 1;
+
+raw_data=$1     # raw data with metadata, txt and wav
+data=$2         # data transformed into kaldi format
+
+# generate kaldi format data for all
+if [ -d ${raw_data} ];then 
+    echo "Generating kaldi format data."
+    mkdir -p $data/data_all
+    find $raw_data -type f -name "*.wav" > $data/data_all/wavpath
+    awk -F'/' '{print $(NF-2)"-"$(NF-1)"-"$NF}' $data/data_all/wavpath | sed 's:\.wav::g' > $data/data_all/uttlist
+    paste $data/data_all/uttlist $data/data_all/wavpath > $data/data_all/wav.scp
+    python local/preprocess.py $data/data_all/wav.scp $data/data_all/trans $data/data_all/utt2spk # faster than for in shell
+    ./utils/utt2spk_to_spk2utt.pl $data/data_all/utt2spk > $data/data_all/spk2utt
+fi
+
+# clean transcription
+if [ -d $data/data_all ];then
+    echo "Cleaning transcription."
+    tr '[a-z]' '[A-Z]' < $data/data_all/trans > $data/data_all/trans_upper
+    # turn "." in specific abbreviations into "<m>" tag
+    sed -i -e 's: MR\.: MR<m>:g' -e 's: MRS\.: MRS<m>:g' -e 's: MS\.: MS<m>:g' \
+        -e 's:^MR\.:MR<m>:g' -e 's:^MRS\.:MRS<m>:g' -e 's:^MS\.:MS<m>:g' $data/data_all/trans_upper 
+	# fix bug
+    sed -i 's:^ST\.:STREET:g' $data/data_all/trans_upper 
+    sed -i 's: ST\.: STREET:g' $data/data_all/trans_upper 
+    # punctuation marks
+    sed -i "s%,\|\.\|?\|!\|;\|-\|:\|,'\|\.'\|?'\|!'\| '% %g" $data/data_all/trans_upper
+    sed -i 's:<m>:.:g' $data/data_all/trans_upper
+    # blank
+    sed -i 's:[ ][ ]*: :g' $data/data_all/trans_upper
+    paste $data/data_all/uttlist $data/data_all/trans_upper > $data/data_all/text
+
+    # critally, must replace tab with space between uttid and text
+    sed -e "s/\t/ /g" -i $data/data_all/text
+fi
+
+echo "local/data_prep.sh succeeded"
+exit 0;

egs/aesrc2020/asr1/local/download_and_untar.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+. ./path.sh || exit 1;
+. ./cmd.sh || exit 1;
+
+zipped_data=$1
+raw_data=$2/Datatang-English/data
+
+# unzip and rename each accent
+unzip $zipped_data -d ${2}
+mv $raw_data/American\ English\ Speech\ Data $raw_data/US
+mv $raw_data/British\ English\ Speech\ Data $raw_data/UK
+mv $raw_data/Chinese\ Speaking\ English\ Speech\ Data $raw_data/CHN 
+mv $raw_data/Indian\ English\ Speech\ Data $raw_data/IND 
+mv $raw_data/Portuguese\ Speaking\ English\ Speech\ Data $raw_data/PT 
+mv $raw_data/Russian\ Speaking\ English\ Speech\ Data $raw_data/RU 
+mv $raw_data/Japanese\ Speaking\ English\ Speech\ Data $raw_data/JPN 
+mv $raw_data/Korean\ Speaking\ English\ Speech\ Data $raw_data/KR
+mv $raw_data/Canadian\ English\ Speech\ Data $raw_data/CA 
+mv $raw_data/Spanish\ Speaking\ English\ Speech\ Data $raw_data/ES
+
+echo "local/download_and_untar.sh succeeded"
+exit 0;

egs/aesrc2020/asr1/local/files/ar.dict

@@ -0,0 +1,8 @@
+<US> 0
+<UK> 1
+<CHN> 2
+<IND> 3
+<JPN> 4
+<KR> 5
+<PT> 6
+<RU> 7

egs/aesrc2020/asr1/local/files/cvlist/CA_cv_spk

@@ -0,0 +1,4 @@
+CA-G00034
+CA-G00086
+CA-G00414
+CA-G20113

egs/aesrc2020/asr1/local/files/cvlist/CHN_cv_spk

@@ -0,0 +1,4 @@
+CHN-G00190
+CHN-G00992
+CHN-G61365
+CHN-G01372

egs/aesrc2020/asr1/local/files/cvlist/ES_cv_spk

@@ -0,0 +1,4 @@
+ES-G00714
+ES-G01878
+ES-G11701
+ES-G20575

egs/aesrc2020/asr1/local/files/cvlist/IND_cv_spk

@@ -0,0 +1,4 @@
+IND-G00892
+IND-G01006
+IND-G01501
+IND-G0760

egs/aesrc2020/asr1/local/files/cvlist/JPN_cv_spk

@@ -0,0 +1,4 @@
+JPN-G00040
+JPN-G00125
+JPN-G00354
+JPN-G20194

egs/aesrc2020/asr1/local/files/cvlist/KR_cv_spk

@@ -0,0 +1,4 @@
+KR-G00022
+KR-G00276
+KR-G10029
+KR-G10122

egs/aesrc2020/asr1/local/files/cvlist/PT_cv_spk

@@ -0,0 +1,5 @@
+PT-G00600
+PT-G00643
+PT-G00963
+PT-G10618
+PT-G20539

egs/aesrc2020/asr1/local/files/cvlist/RU_cv_spk

@@ -0,0 +1,4 @@
+RU-G00163
+RU-G00196
+RU-G00439
+RU-G10416