From 3cac7bb7f732a694f4b87007271d394a9ee3838e Mon Sep 17 00:00:00 2001 From: Chaitanya Narisetty Date: Fri, 13 May 2022 05:07:55 -0400 Subject: [PATCH] resolve conflicts and fix lm_train filenames --- egs2/TEMPLATE/st1/st.sh | 65 +++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/egs2/TEMPLATE/st1/st.sh b/egs2/TEMPLATE/st1/st.sh index bc0f57e4173..03be329cecf 100755 --- a/egs2/TEMPLATE/st1/st.sh +++ b/egs2/TEMPLATE/st1/st.sh @@ -483,7 +483,7 @@ if ! "${skip_data_prep}"; then done utils/combine_data.sh --extra_files "${utt_extra_files}" "data/${train_set}_sp" ${_dirs} for extra_file in ${utt_extra_files}; do - python pyscripts/utils/remove_duplicate_keys.py data/"${train_set}_sp"/${extra_file} > data/"${train_set}_sp"/${extra_file}.tmp + python pyscripts/utils/remove_duplicate_keys.py data/"${train_set}_sp"/${extra_file} > data/"${train_set}_sp"/${extra_file}.tmp mv data/"${train_set}_sp"/${extra_file}.tmp data/"${train_set}_sp"/${extra_file} done else @@ -522,7 +522,7 @@ if ! "${skip_data_prep}"; then for single_file in $(ls data/"${dset}"/${extra_file}*); do cp ${single_file} "${data_feats}${_suf}/${dset}" expand_utt_extra_files="${expand_utt_extra_files} $(basename ${single_file})" - done + done done echo "${expand_utt_extra_files}" utils/fix_data_dir.sh --utt_extra_files "${expand_utt_extra_files}" "${data_feats}${_suf}/${dset}" @@ -567,7 +567,7 @@ if ! "${skip_data_prep}"; then for single_file in $(ls data/"${dset}"/${extra_file}*); do cp ${single_file} "${data_feats}${_suf}/${dset}" expand_utt_extra_files="${expand_utt_extra_files} $(basename ${single_file})" - done + done done for extra_file in ${expand_utt_extra_files}; do LC_ALL=C sort -u -k1,1 "${data_feats}${_suf}/${dset}/${extra_file}" -o "${data_feats}${_suf}/${dset}/${extra_file}" @@ -616,7 +616,7 @@ if ! "${skip_data_prep}"; then for single_file in $(ls data/"${dset}"/${extra_file}*); do cp ${single_file} "${data_feats}${_suf}/${dset}" expand_utt_extra_files="${expand_utt_extra_files} $(basename ${single_file})" - done + done done utils/fix_data_dir.sh --utt_extra_files "${expand_utt_extra_files}*" "${data_feats}${_suf}/${dset}" for extra_file in ${expand_utt_extra_files}; do @@ -706,11 +706,12 @@ if ! "${skip_data_prep}"; then python pyscripts/utils/remove_duplicate_keys.py ${data_feats}/${dset}/${utt_extra_file} \ > ${data_feats}/${dset}/${utt_extra_file}.tmp mv ${data_feats}/${dset}/${utt_extra_file}.tmp ${data_feats}/${dset}/${utt_extra_file} - done + done done # shellcheck disable=SC2002 - cat ${lm_train_text} | awk ' { if( NF != 1 ) print $0; } ' > "${data_feats}/lm_train.${tgt_case}.${tgt_lang}.txt" + cat ${lm_train_text} | awk ' { if( NF != 1 ) print $0; } ' \ + > "${data_feats}/lm_train.${src_lang}.${tgt_case}.${tgt_lang}.txt" fi if [ ${stage} -le 5 ] && [ ${stop_stage} -ge 5 ]; then @@ -783,10 +784,10 @@ if ! "${skip_data_prep}"; then # Create word-list for word-LM training if ${use_word_lm} && [ "${tgt_token_type}" != word ]; then - log "Generate word level token_list from ${data_feats}/lm_train.${tgt_case}.${tgt_lang}.txt" + log "Generate word level token_list from ${data_feats}/lm_train.${src_lang}.${tgt_case}.${tgt_lang}.txt" ${python} -m espnet2.bin.tokenize_text \ --token_type word \ - --input "${data_feats}/lm_train.${tgt_case}.${tgt_lang}.txt" --output "${lm_token_list}" \ + --input "${data_feats}/lm_train.${src_lang}.${tgt_case}.${tgt_lang}.txt" --output "${lm_token_list}" \ --field 2- \ --cleaner "${cleaner}" \ --g2p "${g2p}" \ @@ -872,7 +873,7 @@ fi if ! "${skip_train}"; then if "${use_lm}"; then if [ ${stage} -le 6 ] && [ ${stop_stage} -ge 6 ]; then - log "Stage 6: LM collect stats: train_set=${data_feats}/lm_train.${tgt_case}.${tgt_lang}.txt, dev_set=${lm_dev_text}" + log "Stage 6: LM collect stats: train_set=${data_feats}/lm_train.${src_lang}.${tgt_case}.${tgt_lang}.txt, dev_set=${lm_dev_text}" _opts= if [ -n "${lm_config}" ]; then @@ -885,9 +886,9 @@ if ! "${skip_train}"; then _logdir="${lm_stats_dir}/logdir" mkdir -p "${_logdir}" # Get the minimum number among ${nj} and the number lines of input files - _nj=$(min "${nj}" "$(<${data_feats}/lm_train.${tgt_case}.${tgt_lang}.txt wc -l)" "$(<${lm_dev_text} wc -l)") + _nj=$(min "${nj}" "$(<${data_feats}/lm_train.${src_lang}.${tgt_case}.${tgt_lang}.txt wc -l)" "$(<${lm_dev_text} wc -l)") - key_file="${data_feats}/lm_train.${tgt_case}.${tgt_lang}.txt" + key_file="${data_feats}/lm_train.${src_lang}.${tgt_case}.${tgt_lang}.txt" split_scps="" for n in $(seq ${_nj}); do split_scps+=" ${_logdir}/train.${n}.scp" @@ -911,7 +912,7 @@ if ! "${skip_train}"; then log "LM collect-stats started... log: '${_logdir}/stats.*.log'" # NOTE: --*_shape_file doesn't require length information if --batch_type=unsorted, # but it's used only for deciding the sample ids. - # shellcheck disable=SC2086 + # shellcheck disable=SC2046,SC2086 ${train_cmd} JOB=1:"${_nj}" "${_logdir}"/stats.JOB.log \ ${python} -m espnet2.bin.lm_train \ --collect_stats true \ @@ -922,12 +923,12 @@ if ! "${skip_train}"; then --non_linguistic_symbols "${nlsyms_txt}" \ --cleaner "${cleaner}" \ --g2p "${g2p}" \ - --train_data_path_and_name_and_type "${data_feats}/lm_train.${tgt_case}.${tgt_lang}.txt,text,text" \ + --train_data_path_and_name_and_type "${data_feats}/lm_train.${src_lang}.${tgt_case}.${tgt_lang}.txt,text,text" \ --valid_data_path_and_name_and_type "${lm_dev_text},text,text" \ --train_shape_file "${_logdir}/train.JOB.scp" \ --valid_shape_file "${_logdir}/dev.JOB.scp" \ --output_dir "${_logdir}/stats.JOB" \ - ${_opts} ${lm_args} || { cat "${_logdir}"/stats.1.log; exit 1; } + ${_opts} ${lm_args} || { cat $(grep -l -i error "${_logdir}"/stats.*.log) ; exit 1; } # 4. Aggregate shape files _opts= @@ -949,7 +950,7 @@ if ! "${skip_train}"; then if [ ${stage} -le 7 ] && [ ${stop_stage} -ge 7 ]; then - log "Stage 7: LM Training: train_set=${data_feats}/lm_train.${tgt_case}.${tgt_lang}.txt, dev_set=${lm_dev_text}" + log "Stage 7: LM Training: train_set=${data_feats}/lm_train.${src_lang}.${tgt_case}.${tgt_lang}.txt, dev_set=${lm_dev_text}" _opts= if [ -n "${lm_config}" ]; then @@ -967,7 +968,7 @@ if ! "${skip_train}"; then if [ ! -f "${_split_dir}/.done" ]; then rm -f "${_split_dir}/.done" ${python} -m espnet2.bin.split_scps \ - --scps "${data_feats}/lm_train.${tgt_case}.${tgt_lang}.txt" "${lm_stats_dir}/train/text_shape.${lm_token_type}" \ + --scps "${data_feats}/lm_train.${src_lang}.${tgt_case}.${tgt_lang}.txt" "${lm_stats_dir}/train/text_shape.${lm_token_type}" \ --num_splits "${num_splits_lm}" \ --output_dir "${_split_dir}" touch "${_split_dir}/.done" @@ -975,12 +976,12 @@ if ! "${skip_train}"; then log "${_split_dir}/.done exists. Spliting is skipped" fi - _opts+="--train_data_path_and_name_and_type ${_split_dir}/lm_train.${tgt_case}.${tgt_lang}.txt,text,text " + _opts+="--train_data_path_and_name_and_type ${_split_dir}/lm_train.${src_lang}.${tgt_case}.${tgt_lang}.txt,text,text " _opts+="--train_shape_file ${_split_dir}/text_shape.${lm_token_type} " _opts+="--multiple_iterator true " else - _opts+="--train_data_path_and_name_and_type ${data_feats}/lm_train.${tgt_case}.${tgt_lang}.txt,text,text " + _opts+="--train_data_path_and_name_and_type ${data_feats}/lm_train.${src_lang}.${tgt_case}.${tgt_lang}.txt,text,text " _opts+="--train_shape_file ${lm_stats_dir}/train/text_shape.${lm_token_type} " fi @@ -1053,9 +1054,9 @@ if ! "${skip_train}"; then fi if [ ${stage} -le 9 ] && [ ${stop_stage} -ge 9 ]; then if "${use_ngram}"; then - log "Stage 9: Ngram Training: train_set=${data_feats}/lm_train.${tgt_case}.${tgt_lang}.txt" - cut -f 2 -d " " ${data_feats}/lm_train.${tgt_case}.${tgt_lang}.txt | lmplz -S "20%" --discount_fallback -o ${ngram_num} - >${ngram_exp}/${ngram_num}gram.arpa - build_binary -s ${ngram_exp}/${ngram_num}gram.arpa ${ngram_exp}/${ngram_num}gram.bin + log "Stage 9: Ngram Training: train_set=${data_feats}/lm_train.${src_lang}.${tgt_case}.${tgt_lang}.txt" + cut -f 2 -d " " ${data_feats}/lm_train.${src_lang}.${tgt_case}.${tgt_lang}.txt | lmplz -S "20%" --discount_fallback -o ${ngram_num} - >${ngram_exp}/${ngram_num}gram.arpa + build_binary -s ${ngram_exp}/${ngram_num}gram.arpa ${ngram_exp}/${ngram_num}gram.bin else log "Stage 9: Skip ngram stages: use_ngram=${use_ngram}" fi @@ -1407,7 +1408,7 @@ if ! "${skip_eval}"; then # 2. Submit decoding jobs log "Decoding started... log: '${_logdir}/st_inference.*.log'" - # shellcheck disable=SC2086 + # shellcheck disable=SC2046,SC2086 ${_cmd} --gpu "${_ngpu}" JOB=1:"${_nj}" "${_logdir}"/st_inference.JOB.log \ ${python} -m ${st_inference_tool} \ --batch_size ${batch_size} \ @@ -1417,7 +1418,7 @@ if ! "${skip_eval}"; then --st_train_config "${st_exp}"/config.yaml \ --st_model_file "${st_exp}"/"${inference_st_model}" \ --output_dir "${_logdir}"/output.JOB \ - ${_opts} ${inference_args} + ${_opts} ${inference_args} || { cat $(grep -l -i error "${_logdir}"/st_inference.*.log) ; exit 1; } # 3. Concatenates the output files from each jobs for f in token token_int score text; do @@ -1463,7 +1464,7 @@ if ! "${skip_eval}"; then ) \ <(<"${_data}/utt2spk" awk '{ print "(" $2 "-" $1 ")" }') \ >"${_scoredir}/hyp.trn.org" - + # remove utterance id perl -pe 's/\([^\)]+\)$//g;' "${_scoredir}/ref.trn.org" > "${_scoredir}/ref.trn" perl -pe 's/\([^\)]+\)$//g;' "${_scoredir}/hyp.trn.org" > "${_scoredir}/hyp.trn" @@ -1484,7 +1485,7 @@ if ! "${skip_eval}"; then -i "${_scoredir}/hyp.trn.detok" \ -m bleu chrf ter \ >> ${_scoredir}/result.tc.txt - + log "Write a case-sensitive BLEU (single-reference) result in ${_scoredir}/result.tc.txt" fi @@ -1516,8 +1517,8 @@ if ! "${skip_eval}"; then ) \ <(<"${_data}/utt2spk" awk '{ print "(" $2 "-" $1 ")" }') \ >"${_scoredir}/ref.trn.org.${ref_idx}" - - # + + # remove utterance id perl -pe 's/\([^\)]+\)$//g;' "${_scoredir}/ref.trn.org.${ref_idx}" > "${_scoredir}/ref.trn.${ref_idx}" detokenizer.perl -l ${tgt_lang} -q < "${_scoredir}/ref.trn.${ref_idx}" > "${_scoredir}/ref.trn.detok.${ref_idx}" remove_punctuation.pl < "${_scoredir}/ref.trn.detok.${ref_idx}" > "${_scoredir}/ref.trn.detok.lc.rm.${ref_idx}" @@ -1653,11 +1654,11 @@ if ! "${skip_upload_hf}"; then gitlfs=$(git lfs --version 2> /dev/null || true) [ -z "${gitlfs}" ] && \ log "ERROR: You need to install git-lfs first" && \ - exit 1 - + exit 1 + dir_repo=${expdir}/hf_${hf_repo//"/"/"_"} [ ! -d "${dir_repo}" ] && git clone https://huggingface.co/${hf_repo} ${dir_repo} - + if command -v git &> /dev/null; then _creator_name="$(git config user.name)" _checkout="git checkout $(git show -s --format=%H)" @@ -1670,13 +1671,13 @@ if ! "${skip_upload_hf}"; then # foo/asr1 -> foo _corpus="${_task%/*}" _model_name="${_creator_name}/${_corpus}_$(basename ${packed_model} .zip)" - + # copy files in ${dir_repo} unzip -o ${packed_model} -d ${dir_repo} # Generate description file # shellcheck disable=SC2034 hf_task=speech-translation - # shellcheck disable=SC2034 + # shellcheck disable=SC2034 espnet_task=ST # shellcheck disable=SC2034 task_exp=${st_exp}