-
Notifications
You must be signed in to change notification settings - Fork 287
/
prepare.sh
executable file
·90 lines (82 loc) · 3.7 KB
/
prepare.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
#!/usr/bin/env bash
# fix segmentation fault reported in https://github.com/k2-fsa/icefall/issues/674
export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
set -eou pipefail
nj=15
stage=0
stop_stage=100
. shared/parse_options.sh || exit 1
# All files generated by this script are saved in "data".
# You can safely remove "data" and rerun this script to regenerate it.
mkdir -p data
log() {
# This function is from espnet
local fname=${BASH_SOURCE[1]##*/}
echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
}
if [ $stage -le 0 ] && [ $stop_stage -ge 0 ]; then
log "Stage 0: Prepare wewetspeech dataset."
mkdir -p data/fbank
if [ ! -e data/fbank/.wewetspeech.done ]; then
pushd ../ASR
./prepare.sh --stage 0 --stop-stage 17
./prepare.sh --stage 22 --stop-stage 22
popd
pushd data/fbank
ln -svf $(realpath ../ASR/data/fbank/cuts_DEV.jsonl.gz) .
ln -svf $(realpath ../ASR/data/fbank/feats_DEV.lca) .
ln -svf $(realpath ../ASR/data/fbank/cuts_TEST_NET.jsonl.gz) .
ln -svf $(realpath ../ASR/data/fbank/feats_TEST_NET.lca) .
ln -svf $(realpath ../ASR/data/fbank/cuts_TEST_MEETING.jsonl.gz) .
ln -svf $(realpath ../ASR/data/fbank/feats_TEST_MEETING.lca) .
ln -svf $(realpath ../ASR/data/fbank/cuts_L.jsonl.gz) .
ln -svf $(realpath ../ASR/data/fbank/L_split_1000) .
ln -svf $(realpath ../ASR/data/fbank/cuts_M.jsonl.gz) .
ln -svf $(realpath ../ASR/data/fbank/M_split_1000) .
ln -svf $(realpath ../ASR/data/fbank/cuts_S.jsonl.gz) .
ln -svf $(realpath ../ASR/data/fbank/S_split_1000) .
ln -svf $(realpath ../ASR/data/fbank/musan_cuts.jsonl.gz) .
ln -svf $(realpath ../ASR/data/fbank/musan_feats) .
popd
pushd data
ln -svf $(realpath ../ASR/data/lang_partial_tone) .
popd
touch data/fbank/.wewetspeech.done
else
log "WenetSpeech dataset already exists, skipping."
fi
fi
if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then
log "Stage 1: Prepare open commands dataset."
mkdir -p data/fbank
if [ ! -e data/fbank/.cn_speech_commands.done ]; then
pushd data
git clone https://github.com/pkufool/open-commands.git
ln -svf $(realpath ./open-commands/CN/small/commands.txt) commands_small.txt
ln -svf $(realpath ./open-commands/CN/large/commands.txt) commands_large.txt
pushd open-commands
./script/prepare.sh --stage 1 --stop-stage 1
./script/prepare.sh --stage 3 --stop-stage 5
popd
popd
pushd data/fbank
ln -svf $(realpath ../open-commands/data/fbank/cn_speech_commands_cuts_large.jsonl.gz) .
ln -svf $(realpath ../open-commands/data/fbank/cn_speech_commands_feats_large) .
ln -svf $(realpath ../open-commands/data/fbank/cn_speech_commands_cuts_small.jsonl.gz) .
ln -svf $(realpath ../open-commands/data/fbank/cn_speech_commands_feats_small) .
ln -svf $(realpath ../open-commands/data/fbank/nihaowenwen_cuts_dev.jsonl.gz) .
ln -svf $(realpath ../open-commands/data/fbank/nihaowenwen_feats_dev) .
ln -svf $(realpath ../open-commands/data/fbank/nihaowenwen_cuts_test.jsonl.gz) .
ln -svf $(realpath ../open-commands/data/fbank/nihaowenwen_feats_test) .
ln -svf $(realpath ../open-commands/data/fbank/nihaowenwen_cuts_train.jsonl.gz) .
ln -svf $(realpath ../open-commands/data/fbank/nihaowenwen_feats_train) .
ln -svf $(realpath ../open-commands/data/fbank/xiaoyun_cuts_clean.jsonl.gz) .
ln -svf $(realpath ../open-commands/data/fbank/xiaoyun_feats_clean.lca) .
ln -svf $(realpath ../open-commands/data/fbank/xiaoyun_cuts_noisy.jsonl.gz) .
ln -svf $(realpath ../open-commands/data/fbank/xiaoyun_feats_noisy.lca) .
popd
touch data/fbank/.cn_speech_commands.done
else
log "CN speech commands dataset already exists, skipping."
fi
fi