From 3b53aedc654fd30a828689c2139a1e130adac077 Mon Sep 17 00:00:00 2001
From: YushiUeda <yueda@br013.ib.bridges2.psc.edu>
Date: Fri, 25 Feb 2022 00:13:16 -0500
Subject: [PATCH 1/7] initial commit

---
 egs2/swbd_sentiment/asr1/asr.sh               |   1 +
 egs2/swbd_sentiment/asr1/cmd.sh               | 110 ++++++++++++++++
 egs2/swbd_sentiment/asr1/conf/fbank.conf      |   2 +
 egs2/swbd_sentiment/asr1/conf/pbs.conf        |  11 ++
 egs2/swbd_sentiment/asr1/conf/pitch.conf      |   1 +
 egs2/swbd_sentiment/asr1/conf/queue.conf      |  12 ++
 egs2/swbd_sentiment/asr1/conf/slurm.conf      |  14 ++
 egs2/swbd_sentiment/asr1/db.sh                |   1 +
 .../asr1/local/MSU_single_letter.txt          |   1 +
 egs2/swbd_sentiment/asr1/local/data.sh        |  83 ++++++++++++
 egs2/swbd_sentiment/asr1/local/dict.patch     |   1 +
 .../asr1/local/extend_segments.pl             |   1 +
 .../asr1/local/format_acronyms_dict.py        |   1 +
 .../asr1/local/map_acronyms_transcripts.py    |   1 +
 egs2/swbd_sentiment/asr1/local/path.sh        |   0
 .../asr1/local/prepare_sentiment.py           | 122 ++++++++++++++++++
 egs2/swbd_sentiment/asr1/local/score.sh       | 116 +++++++++++++++++
 .../asr1/local/swbd1_data_download.sh         |   1 +
 .../asr1/local/swbd1_data_prep.sh             |   1 +
 .../asr1/local/swbd1_fix_speakerid.pl         |   1 +
 .../asr1/local/swbd1_map_words.pl             |   1 +
 .../asr1/local/swbd1_prepare_dict.sh          |   1 +
 egs2/swbd_sentiment/asr1/path.sh              |   1 +
 egs2/swbd_sentiment/asr1/pyscripts            |   1 +
 egs2/swbd_sentiment/asr1/scripts              |   1 +
 egs2/swbd_sentiment/asr1/steps                |   1 +
 egs2/swbd_sentiment/asr1/utils                |   1 +
 27 files changed, 488 insertions(+)
 create mode 120000 egs2/swbd_sentiment/asr1/asr.sh
 create mode 100644 egs2/swbd_sentiment/asr1/cmd.sh
 create mode 100644 egs2/swbd_sentiment/asr1/conf/fbank.conf
 create mode 100644 egs2/swbd_sentiment/asr1/conf/pbs.conf
 create mode 100644 egs2/swbd_sentiment/asr1/conf/pitch.conf
 create mode 100644 egs2/swbd_sentiment/asr1/conf/queue.conf
 create mode 100644 egs2/swbd_sentiment/asr1/conf/slurm.conf
 create mode 120000 egs2/swbd_sentiment/asr1/db.sh
 create mode 120000 egs2/swbd_sentiment/asr1/local/MSU_single_letter.txt
 create mode 100755 egs2/swbd_sentiment/asr1/local/data.sh
 create mode 120000 egs2/swbd_sentiment/asr1/local/dict.patch
 create mode 120000 egs2/swbd_sentiment/asr1/local/extend_segments.pl
 create mode 120000 egs2/swbd_sentiment/asr1/local/format_acronyms_dict.py
 create mode 120000 egs2/swbd_sentiment/asr1/local/map_acronyms_transcripts.py
 create mode 100644 egs2/swbd_sentiment/asr1/local/path.sh
 create mode 100755 egs2/swbd_sentiment/asr1/local/prepare_sentiment.py
 create mode 100755 egs2/swbd_sentiment/asr1/local/score.sh
 create mode 120000 egs2/swbd_sentiment/asr1/local/swbd1_data_download.sh
 create mode 120000 egs2/swbd_sentiment/asr1/local/swbd1_data_prep.sh
 create mode 120000 egs2/swbd_sentiment/asr1/local/swbd1_fix_speakerid.pl
 create mode 120000 egs2/swbd_sentiment/asr1/local/swbd1_map_words.pl
 create mode 120000 egs2/swbd_sentiment/asr1/local/swbd1_prepare_dict.sh
 create mode 120000 egs2/swbd_sentiment/asr1/path.sh
 create mode 120000 egs2/swbd_sentiment/asr1/pyscripts
 create mode 120000 egs2/swbd_sentiment/asr1/scripts
 create mode 120000 egs2/swbd_sentiment/asr1/steps
 create mode 120000 egs2/swbd_sentiment/asr1/utils

diff --git a/egs2/swbd_sentiment/asr1/asr.sh b/egs2/swbd_sentiment/asr1/asr.sh
new file mode 120000
index 00000000000..60b05122cfd
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/asr.sh
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/asr.sh
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/cmd.sh b/egs2/swbd_sentiment/asr1/cmd.sh
new file mode 100644
index 00000000000..2aae6919fef
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/cmd.sh
@@ -0,0 +1,110 @@
+# ====== About run.pl, queue.pl, slurm.pl, and ssh.pl ======
+# Usage: <cmd>.pl [options] JOB=1:<nj> <log> <command...>
+# e.g.
+#   run.pl --mem 4G JOB=1:10 echo.JOB.log echo JOB
+#
+# Options:
+#   --time <time>: Limit the maximum time to execute.
+#   --mem <mem>: Limit the maximum memory usage.
+#   -–max-jobs-run <njob>: Limit the number parallel jobs. This is ignored for non-array jobs.
+#   --num-threads <ngpu>: Specify the number of CPU core.
+#   --gpu <ngpu>: Specify the number of GPU devices.
+#   --config: Change the configuration file from default.
+#
+# "JOB=1:10" is used for "array jobs" and it can control the number of parallel jobs.
+# The left string of "=", i.e. "JOB", is replaced by <N>(Nth job) in the command and the log file name,
+# e.g. "echo JOB" is changed to "echo 3" for the 3rd job and "echo 8" for 8th job respectively.
+# Note that the number must start with a positive number, so you can't use "JOB=0:10" for example.
+#
+# run.pl, queue.pl, slurm.pl, and ssh.pl have unified interface, not depending on its backend.
+# These options are mapping to specific options for each backend and
+# it is configured by "conf/queue.conf" and "conf/slurm.conf" by default.
+# If jobs failed, your configuration might be wrong for your environment.
+#
+#
+# The official documentation for run.pl, queue.pl, slurm.pl, and ssh.pl:
+#   "Parallelization in Kaldi": http://kaldi-asr.org/doc/queue.html
+# =========================================================~
+
+
+# Select the backend used by run.sh from "local", "stdout", "sge", "slurm", or "ssh"
+cmd_backend='local'
+
+# Local machine, without any Job scheduling system
+if [ "${cmd_backend}" = local ]; then
+
+    # The other usage
+    export train_cmd="run.pl"
+    # Used for "*_train.py": "--gpu" is appended optionally by run.sh
+    export cuda_cmd="run.pl"
+    # Used for "*_recog.py"
+    export decode_cmd="run.pl"
+
+# Local machine logging to stdout and log file, without any Job scheduling system
+elif [ "${cmd_backend}" = stdout ]; then
+
+    # The other usage
+    export train_cmd="stdout.pl"
+    # Used for "*_train.py": "--gpu" is appended optionally by run.sh
+    export cuda_cmd="stdout.pl"
+    # Used for "*_recog.py"
+    export decode_cmd="stdout.pl"
+
+
+# "qsub" (Sun Grid Engine, or derivation of it)
+elif [ "${cmd_backend}" = sge ]; then
+    # The default setting is written in conf/queue.conf.
+    # You must change "-q g.q" for the "queue" for your environment.
+    # To know the "queue" names, type "qhost -q"
+    # Note that to use "--gpu *", you have to setup "complex_value" for the system scheduler.
+
+    export train_cmd="queue.pl"
+    export cuda_cmd="queue.pl"
+    export decode_cmd="queue.pl"
+
+
+# "qsub" (Torque/PBS.)
+elif [ "${cmd_backend}" = pbs ]; then
+    # The default setting is written in conf/pbs.conf.
+
+    export train_cmd="pbs.pl"
+    export cuda_cmd="pbs.pl"
+    export decode_cmd="pbs.pl"
+
+
+# "sbatch" (Slurm)
+elif [ "${cmd_backend}" = slurm ]; then
+    # The default setting is written in conf/slurm.conf.
+    # You must change "-p cpu" and "-p gpu" for the "partition" for your environment.
+    # To know the "partion" names, type "sinfo".
+    # You can use "--gpu * " by default for slurm and it is interpreted as "--gres gpu:*"
+    # The devices are allocated exclusively using "${CUDA_VISIBLE_DEVICES}".
+
+    export train_cmd="slurm.pl"
+    export cuda_cmd="slurm.pl"
+    export decode_cmd="slurm.pl"
+
+elif [ "${cmd_backend}" = ssh ]; then
+    # You have to create ".queue/machines" to specify the host to execute jobs.
+    # e.g. .queue/machines
+    #   host1
+    #   host2
+    #   host3
+    # Assuming you can login them without any password, i.e. You have to set ssh keys.
+
+    export train_cmd="ssh.pl"
+    export cuda_cmd="ssh.pl"
+    export decode_cmd="ssh.pl"
+
+# This is an example of specifying several unique options in the JHU CLSP cluster setup.
+# Users can modify/add their own command options according to their cluster environments.
+elif [ "${cmd_backend}" = jhu ]; then
+
+    export train_cmd="queue.pl --mem 2G"
+    export cuda_cmd="queue-freegpu.pl --mem 2G --gpu 1 --config conf/queue.conf"
+    export decode_cmd="queue.pl --mem 4G"
+
+else
+    echo "$0: Error: Unknown cmd_backend=${cmd_backend}" 1>&2
+    return 1
+fi
diff --git a/egs2/swbd_sentiment/asr1/conf/fbank.conf b/egs2/swbd_sentiment/asr1/conf/fbank.conf
new file mode 100644
index 00000000000..82ac7bd0dbc
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/conf/fbank.conf
@@ -0,0 +1,2 @@
+--sample-frequency=16000 
+--num-mel-bins=80
diff --git a/egs2/swbd_sentiment/asr1/conf/pbs.conf b/egs2/swbd_sentiment/asr1/conf/pbs.conf
new file mode 100644
index 00000000000..119509938ce
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/conf/pbs.conf
@@ -0,0 +1,11 @@
+# Default configuration
+command qsub -V -v PATH -S /bin/bash
+option name=* -N $0
+option mem=* -l mem=$0
+option mem=0          # Do not add anything to qsub_opts
+option num_threads=* -l ncpus=$0
+option num_threads=1  # Do not add anything to qsub_opts
+option num_nodes=* -l nodes=$0:ppn=1
+default gpu=0
+option gpu=0
+option gpu=* -l ngpus=$0
diff --git a/egs2/swbd_sentiment/asr1/conf/pitch.conf b/egs2/swbd_sentiment/asr1/conf/pitch.conf
new file mode 100644
index 00000000000..e959a19d5b8
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/conf/pitch.conf
@@ -0,0 +1 @@
+--sample-frequency=16000
diff --git a/egs2/swbd_sentiment/asr1/conf/queue.conf b/egs2/swbd_sentiment/asr1/conf/queue.conf
new file mode 100644
index 00000000000..500582fab31
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/conf/queue.conf
@@ -0,0 +1,12 @@
+# Default configuration
+command qsub -v PATH -cwd -S /bin/bash -j y -l arch=*64*
+option name=* -N $0
+option mem=* -l mem_free=$0,ram_free=$0
+option mem=0          # Do not add anything to qsub_opts
+option num_threads=* -pe smp $0
+option num_threads=1  # Do not add anything to qsub_opts
+option max_jobs_run=* -tc $0
+option num_nodes=* -pe mpi $0  # You must set this PE as allocation_rule=1
+default gpu=0
+option gpu=0
+option gpu=* -l gpu=$0 -q g.q
diff --git a/egs2/swbd_sentiment/asr1/conf/slurm.conf b/egs2/swbd_sentiment/asr1/conf/slurm.conf
new file mode 100644
index 00000000000..3b229673638
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/conf/slurm.conf
@@ -0,0 +1,14 @@
+# Default configuration
+command sbatch --export=PATH
+option name=* --job-name $0
+option time=* --time $0
+option mem=* --mem-per-cpu $0
+option mem=0
+option num_threads=* --cpus-per-task $0
+option num_threads=1 --cpus-per-task 1
+option num_nodes=* --nodes $0
+default gpu=0
+option gpu=0 -p cpu
+option gpu=* -p gpu --gres=gpu:$0 -c $0  # Recommend allocating more CPU than, or equal to the number of GPU
+# note: the --max-jobs-run option is supported as a special case
+# by slurm.pl and you don't have to handle it in the config file.
diff --git a/egs2/swbd_sentiment/asr1/db.sh b/egs2/swbd_sentiment/asr1/db.sh
new file mode 120000
index 00000000000..50d86130898
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/db.sh
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/db.sh
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/local/MSU_single_letter.txt b/egs2/swbd_sentiment/asr1/local/MSU_single_letter.txt
new file mode 120000
index 00000000000..dd1bbcd661f
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/local/MSU_single_letter.txt
@@ -0,0 +1 @@
+../../../../egs/swbd/asr1/local/MSU_single_letter.txt
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/local/data.sh b/egs2/swbd_sentiment/asr1/local/data.sh
new file mode 100755
index 00000000000..ce85bbf5479
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/local/data.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+log() {
+    local fname=${BASH_SOURCE[1]##*/}
+    echo -e "$(date '+%Y-%m-%dT%H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
+}
+SECONDS=0
+
+
+stage=1
+stop_stage=4
+
+log "$0 $*"
+. utils/parse_options.sh
+
+. ./db.sh
+. ./path.sh
+. ./cmd.sh
+
+# for local usage. Delete when making a PR.
+SWBD="swbd"
+
+if [ $# -ne 0 ]; then
+    log "Error: No positional arguments are required."
+    exit 2
+fi
+
+if [ -z "${SWBD}" ]; then
+    log "Fill the value of 'SWBD' of db.sh"
+    exit 1
+fi
+
+
+# we assume that LDC97S62 & speech_sentiment_annotations are placed under SWBD
+swbd1_dir=${SWBD}/LDC97S62
+swbd_sentiment=${SWBD}/speech_sentiment_annotations/data/sentiment_labels.tsv
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    log " Data Preparation"
+    local/swbd1_data_download.sh ${swbd1_dir}
+    local/swbd1_prepare_dict.sh
+    local/swbd1_data_prep.sh ${swbd1_dir}
+    # upsample audio from 8k to 16k to make a recipe consistent with others
+    sed -i.bak -e "s/$/ sox -R -t wav - -t wav - rate 16000 dither | /" data/train/wav.scp
+    utils/fix_data_dir.sh data/train
+fi
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+    log " Data Formatting"
+     # remove ._ . _1 symbols from text  
+     cp data/train/text data/train/text.backup
+     sed -i 's/\._/ /g; s/\.//g; s/them_1/them/g' data/train/text
+fi
+
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+    log " Concatenate Sentiment with Transcript"
+    # concat sentiment (Positive, Negative, Neutral). 
+    # Using annotation reconciliation strategy based on majority voting as in
+    # https://catalog.ldc.upenn.edu/docs/LDC2020T14/LREC_2020_Switchboard_Senti.pdf
+    # this stage may take a while
+    mkdir -p data/local/tmp/
+    mv -f data/train/* data/local/tmp/.
+    mkdir -p data/dev/
+    mkdir -p data/test/
+    python3 local/prepare_sentiment.py \
+        --train_dir data/train/ \
+        --dev_dir data/dev/ \
+        --test_dir data/test/ \
+        --sentiment_file ${swbd_sentiment} \
+        --text_file data/local/tmp/text \
+        --wavscp_file data/local/tmp/wav.scp
+    for dir in train dev test; do
+    utils/utt2spk_to_spk2utt.pl data/$dir/utt2spk > data/$dir/spk2utt
+    utils/validate_data_dir.sh data/$dir
+    done
+fi
+
+log "Successfully finished. [elapsed=${SECONDS}s]"
diff --git a/egs2/swbd_sentiment/asr1/local/dict.patch b/egs2/swbd_sentiment/asr1/local/dict.patch
new file mode 120000
index 00000000000..f3e0d14e91a
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/local/dict.patch
@@ -0,0 +1 @@
+../../../../egs/swbd/asr1/local/dict.patch
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/local/extend_segments.pl b/egs2/swbd_sentiment/asr1/local/extend_segments.pl
new file mode 120000
index 00000000000..63065555357
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/local/extend_segments.pl
@@ -0,0 +1 @@
+../../../../egs/swbd/asr1/local/extend_segments.pl
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/local/format_acronyms_dict.py b/egs2/swbd_sentiment/asr1/local/format_acronyms_dict.py
new file mode 120000
index 00000000000..29f6093eade
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/local/format_acronyms_dict.py
@@ -0,0 +1 @@
+../../../../egs/swbd/asr1/local/format_acronyms_dict.py
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/local/map_acronyms_transcripts.py b/egs2/swbd_sentiment/asr1/local/map_acronyms_transcripts.py
new file mode 120000
index 00000000000..1d1cda1cc86
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/local/map_acronyms_transcripts.py
@@ -0,0 +1 @@
+../../../../egs/swbd/asr1/local/map_acronyms_transcripts.py
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/local/path.sh b/egs2/swbd_sentiment/asr1/local/path.sh
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/egs2/swbd_sentiment/asr1/local/prepare_sentiment.py b/egs2/swbd_sentiment/asr1/local/prepare_sentiment.py
new file mode 100755
index 00000000000..331271efa7d
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/local/prepare_sentiment.py
@@ -0,0 +1,122 @@
+import os
+import re
+import argparse
+import math
+
+def float2str(number, size=6):
+    number = str(math.ceil(number * 100))
+    return (size - len(number)) * "0" + number
+
+def majorityvote(line):
+    count_pos = line.count("Positive")
+    count_neu = line.count("Neutral")
+    count_neg = line.count("Negative")
+    dic = {"Positive": count_pos, "Neutral": count_neu, "Negative": count_neg}
+    max_value = max(dic.values())
+    # make sure max_value is unique
+    keys = [key for key, value in dic.items() if value == max_value]
+    label = keys[0] if len(keys) == 1 else -1
+    return label
+
+def normalize_transcript(transcript):
+    # remove punctuation except apostrophes
+    transcript = re.sub('(\.|\,|\?|\!|\-|\:|\;)', ' \\1 ', transcript)
+    transcript = re.sub('\.|\,|\?|\!|\-|\:|\;', '', transcript)
+    # remove tag (e.g. [LAUGHTER])
+    transcript = re.sub('\[.+\]', '', transcript)
+    # Detect valid apostrophe cases and split those into two words
+    transcript = re.sub('([a-z])\'([a-z])', '\\1 \'\\2', transcript)
+    # Clean up special cases of standalone apostrophes
+    transcript = re.sub('([a-z])\' ', '\\1 ', transcript)
+    # remove extra spaces
+    transcript = re.sub(' +', ' ', transcript)
+    # remove space at the beginning of the utterance
+    transcript = re.sub('^ ', '', transcript)
+    return transcript
+
+def process_data(target_dir, sentiment_file, text_file, wavscp_file, start_linenum, end_linenum):
+    if not os.path.exists(target_dir):
+        os.makedirs(target_dir)
+
+    utt2spk_list = []
+    segments_list = []
+    text_list = []
+    wavscp_list = []
+    reco2file_list = []
+
+    with open(sentiment_file, "r", encoding="utf-8") as sf, open(text_file, "r", encoding="utf-8") as tf, open(wavscp_file, "r", encoding="utf-8") as wf:
+        prev_spk_id_tf = 0
+        prev_linenum_tf = 0
+        prev_linenum_wf = 0
+        for linenum, line_sf in enumerate(sf):
+            if linenum >= start_linenum and linenum < end_linenum:
+                # "sw02005_0[tab]0.0[tab]11.287375[tab]Neutral-{Questioning}#Neutral-{No emotion}#Neutral-{No emotion}"
+                utt_id_sf, start, end, sentiment = line_sf.strip().split("\t")
+                # "sw02005_0" -> "sw02005"
+                reco_id_sf = utt_id_sf.split("_")[0]
+                label = majorityvote(sentiment)
+                if label != -1:
+                    tf.seek(0)
+                    for linenum_tf, line_tf in enumerate(tf):
+                        if linenum_tf >= prev_linenum_tf:
+                            # "sw02001-A_018732-018950 oh i see uh-huh" -> "sw02001-A_018732-018950" "oh i see uh-huh"
+                            utt_id_tf, transcript = line_tf.strip("\n").split(" ", 1)
+                            # "sw02001-A_018732-018950" -> "sw02001-A" "018732-018950"
+                            spk_id_tf, time_id = utt_id_tf.split("_")
+                            # "sw02001-A" -> "sw02001"
+                            reco_id_tf = spk_id_tf.split("-")[0]
+                            # "018732-018950" -> "018732" "018950"
+                            start_time_id, end_time_id = time_id.split("-")
+                            # in case start and end time slightly differ in text and sentiment annotation
+                            eps = 0.05
+                            if reco_id_tf == reco_id_sf and start_time_id >= float2str(float(start) - eps) and start_time_id <= float2str(float(start) + eps) and end_time_id >= float2str(float(end) - eps) and end_time_id <= float2str(float(end) + eps):
+                                #print("{} {} {} {}".format(utt_id_tf, reco_id_sf, start_time_id, end_time_id))
+                                # normalize transcript
+                                transcript = normalize_transcript(transcript)
+                                utt2spk_list.append("{} {}".format(utt_id_tf, spk_id_tf))
+                                segments_list.append("{} {:.2f} {:.2f}".format(utt_id_tf, float(start), float(end)))
+                                text_list.append("{} {} {}".format(utt_id_tf, label, transcript))
+
+                                if prev_spk_id_tf != spk_id_tf:
+                                    wf.seek(0)
+                                    for linenum_wf, line_wf in enumerate(wf):
+                                        if linenum_wf >= prev_linenum_wf:
+                                            spk_id_wf = line_wf.split(" ")[0]
+                                            if spk_id_wf == spk_id_tf:
+                                                wavscp_list.append("{}".format(line_wf.strip("\n")))
+                                                reco_id_wf, channel_id = spk_id_wf.split("-")
+                                                reco2file_list.append("{} {} {}".format(spk_id_wf, reco_id_wf, channel_id))
+                                                prev_linenum_wf = linenum_wf
+                                                break
+                                prev_spk_id_tf = spk_id_tf
+                                prev_linenum_tf = linenum_tf
+                                break
+    with open(os.path.join(target_dir,"utt2spk"), "w", encoding="utf-8") as utt2spk, open(os.path.join(target_dir,"segments"), "w", encoding="utf-8") as segments, open(os.path.join(target_dir,"text"), "w", encoding="utf-8") as text, open(os.path.join(target_dir,"wav.scp"), "w", encoding="utf-8") as wavscp, open(os.path.join(target_dir,"reco2file_and_channel"), "w", encoding="utf-8") as reco2file:
+        utt2spk.write("\n".join(utt2spk_list)+"\n")
+        segments.write("\n".join(segments_list)+"\n")
+        text.write("\n".join(text_list)+"\n")
+        wavscp.write("\n".join(wavscp_list)+"\n")
+        reco2file.write("\n".join(reco2file_list)+"\n")
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--train_dir", type=str, default="data/train")
+parser.add_argument("--dev_dir", type=str, default="data/dev")
+parser.add_argument("--test_dir", type=str, default="data/test")
+parser.add_argument("--sentiment_file", type=str, required=True)
+parser.add_argument("--text_file", type=str, default="data/train/tmp/text")
+parser.add_argument("--wavscp_file", type=str, default="data/train/tmp/wav.scp")
+
+args = parser.parse_args()
+
+# Split into train, dev, test
+# Note that there is no "official" split provided. 
+# Using the proportion of train 90%, dev 5%, test 5% as in
+# https://arxiv.org/pdf/1911.09762.pdf
+print("start train file preparation")
+process_data(args.train_dir, args.sentiment_file, args.text_file, args.wavscp_file, 0, 47056)
+print("start dev file preparation")
+process_data(args.dev_dir, args.sentiment_file, args.text_file, args.wavscp_file, 47056, 49673)
+print("start test file preparation")
+process_data(args.test_dir, args.sentiment_file, args.text_file, args.wavscp_file, 49673, 52293)
+
+print("Successfully finished text, utt2spk, segments, wavescp preparation")
diff --git a/egs2/swbd_sentiment/asr1/local/score.sh b/egs2/swbd_sentiment/asr1/local/score.sh
new file mode 100755
index 00000000000..a0316ac7c49
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/local/score.sh
@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+# Copyright Johns Hopkins University (Author: Daniel Povey) 2012.  Apache 2.0.
+
+# begin configuration section.
+cmd=run.pl
+stage=0
+data=data/eval2000
+#end configuration section.
+
+[ -f ./path.sh ] && . ./path.sh
+. parse_options.sh || exit 1;
+
+if [ $# -lt 1 ]; then
+  echo "Usage: local/score.sh [--cmd (run.pl|queue.pl...)] <asr-exp-dir>"
+  echo " Options:"
+  echo "    --cmd (run.pl|queue.pl...)      # specify how to run the sub-processes."
+  echo "    --stage (0|1|2)                 # start scoring script from part-way through."
+  exit 1;
+fi
+
+asr_expdir=$1
+
+hubscr=${KALDI_ROOT}/tools/sctk/bin/hubscr.pl
+[ ! -f ${hubscr} ] && echo "Cannot find scoring program at $hubscr" && exit 1;
+hubdir=$(dirname ${hubscr})
+
+for f in ${data}/stm ${data}/glm; do
+  [ ! -f ${f} ] && echo "$0: expecting file $f to exist" && exit 1;
+done
+name=$(basename ${data}) # e.g. eval2000
+for dir in ${asr_expdir}/decode_*/${name}/score_wer; do
+    score_dir=${dir}/scoring
+    ctm=${score_dir}/hyp.ctm
+    # The WER seems to be lower without converting stm
+    #stm=${score_dir}/ref.stm
+    stm=${data}/stm
+    mkdir -p ${score_dir}
+    if [ ${stage} -le 0 ]; then
+        
+            # Assuming trn files exist
+            #ref=${dir}/ref.wrd.trn
+            hyp=${dir}/hyp.trn
+            # The WER seems to be lower without converting stm
+            #trn2stm.py --orig-stm ${data}/stm ${ref} ${stm}
+            trn2ctm.py ${hyp} ${ctm}
+        
+    fi
+
+    if [ ${stage} -le 1 ]; then
+      # Remove some stuff we don't want to score, from the ctm.
+      # the big expression in parentheses contains all the things that get mapped
+      # by the glm file, into hesitations.
+      # The -$ expression removes partial words.
+      # the aim here is to remove all the things that appear in the reference as optionally
+      # deletable (inside parentheses), as if we delete these there is no loss, while
+      # if we get them correct there is no gain.
+      cp ${ctm} ${score_dir}/tmpf;
+      cat ${score_dir}/tmpf | grep -i -v -E '\[NOISE|LAUGHTER|VOCALIZED-NOISE\]' | \
+      grep -i -v -E '<UNK>' | \
+      grep -i -v -E ' (UH|UM|EH|MM|HM|AH|HUH|HA|ER|OOF|HEE|ACH|EEE|EW)$' | \
+      grep -v -- '-$' > ${ctm};
+    fi
+
+    # Score the set...
+    if [ ${stage} -le 2 ]; then
+        ${cmd} ${score_dir}/score.log ${hubscr} -p ${hubdir} -V -l english -h hub5 -g ${data}/glm -r ${stm} ${ctm} || exit 1;
+    fi
+
+    # For eval2000 score the subsets
+    case "$name" in
+      eval2000*)
+        # Score only the, swbd part...
+        if [ ${stage} -le 3 ]; then
+            swbd_stm=${score_dir}/ref.swbd.stm
+            swbd_ctm=${score_dir}/hyp.swbd.ctm
+            ${cmd} ${score_dir}/score.swbd.log \
+              grep -v '^en_' ${stm} '>' ${swbd_stm} '&&' \
+              grep -v '^en_' ${ctm} '>' ${swbd_ctm} '&&' \
+              ${hubscr} -p ${hubdir} -V -l english -h hub5 -g ${data}/glm -r ${swbd_stm} ${swbd_ctm} || exit 1;
+        fi
+        # Score only the, callhome part...
+        if [ ${stage} -le 3 ]; then
+            callhm_stm=${score_dir}/ref.callhm.stm
+            callhm_ctm=${score_dir}/hyp.callhm.ctm
+            ${cmd} ${score_dir}/score.callhm.log \
+              grep -v '^sw_' ${stm} '>' ${callhm_stm} '&&' \
+              grep -v '^sw_' ${ctm} '>' ${callhm_ctm} '&&' \
+              ${hubscr} -p ${hubdir} -V -l english -h hub5 -g ${data}/glm -r ${callhm_stm} ${callhm_ctm} || exit 1;
+        fi
+        ;;
+    rt03* )
+
+      # Score only the swbd part...
+      if [ ${stage} -le 3 ]; then
+            swbd_stm=${score_dir}/ref.swbd.stm
+            swbd_ctm=${score_dir}/hyp.swbd.ctm
+            ${cmd} ${score_dir}/score.swbd.log \
+              grep -v '^fsh_' ${stm} '>' ${swbd_stm} '&&' \
+              grep -v '^fsh_' ${ctm} '>' ${swbd_ctm} '&&' \
+              ${hubscr} -p ${hubdir} -V -l english -h hub5 -g ${data}/glm -r ${swbd_stm} ${swbd_ctm} || exit 1;
+      fi
+      # Score only the fisher part...
+      if [ ${stage} -le 3 ]; then
+            fsh_stm=${score_dir}/ref.fsh.stm
+            fsh_ctm=${score_dir}/hyp.fsh.ctm
+            ${cmd} ${score_dir}/score.fsh.log \
+              grep -v '^sw_' ${stm} '>' ${fsh_stm} '&&' \
+              grep -v '^sw_' ${ctm} '>' ${fsh_ctm} '&&' \
+              ${hubscr} -p ${hubdir} -V -l english -h hub5 -g ${data}/glm -r ${fsh_stm} ${fsh_ctm} || exit 1;
+      fi
+    ;;
+    esac
+
+    grep 'Percent Total Error' ${score_dir}/hyp.*ctm.filt.dtl
+done
+exit 0
diff --git a/egs2/swbd_sentiment/asr1/local/swbd1_data_download.sh b/egs2/swbd_sentiment/asr1/local/swbd1_data_download.sh
new file mode 120000
index 00000000000..dfc7b6be51e
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/local/swbd1_data_download.sh
@@ -0,0 +1 @@
+../../../../egs/swbd/asr1/local/swbd1_data_download.sh
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/local/swbd1_data_prep.sh b/egs2/swbd_sentiment/asr1/local/swbd1_data_prep.sh
new file mode 120000
index 00000000000..2c88651a694
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/local/swbd1_data_prep.sh
@@ -0,0 +1 @@
+../../../../egs/swbd/asr1/local/swbd1_data_prep.sh
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/local/swbd1_fix_speakerid.pl b/egs2/swbd_sentiment/asr1/local/swbd1_fix_speakerid.pl
new file mode 120000
index 00000000000..895ea088e4a
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/local/swbd1_fix_speakerid.pl
@@ -0,0 +1 @@
+../../../../egs/swbd/asr1/local/swbd1_fix_speakerid.pl
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/local/swbd1_map_words.pl b/egs2/swbd_sentiment/asr1/local/swbd1_map_words.pl
new file mode 120000
index 00000000000..7c0014e683a
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/local/swbd1_map_words.pl
@@ -0,0 +1 @@
+../../../../egs/swbd/asr1/local/swbd1_map_words.pl
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/local/swbd1_prepare_dict.sh b/egs2/swbd_sentiment/asr1/local/swbd1_prepare_dict.sh
new file mode 120000
index 00000000000..88fa6f959b1
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/local/swbd1_prepare_dict.sh
@@ -0,0 +1 @@
+../../../../egs/swbd/asr1/local/swbd1_prepare_dict.sh
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/path.sh b/egs2/swbd_sentiment/asr1/path.sh
new file mode 120000
index 00000000000..c9ac0a75bc6
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/path.sh
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/path.sh
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/pyscripts b/egs2/swbd_sentiment/asr1/pyscripts
new file mode 120000
index 00000000000..ac68ad75b60
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/pyscripts
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/pyscripts
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/scripts b/egs2/swbd_sentiment/asr1/scripts
new file mode 120000
index 00000000000..b25829705dc
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/scripts
@@ -0,0 +1 @@
+../../TEMPLATE/asr1/scripts
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/steps b/egs2/swbd_sentiment/asr1/steps
new file mode 120000
index 00000000000..91f2d234e20
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/steps
@@ -0,0 +1 @@
+../../../tools/kaldi/egs/wsj/s5/steps
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/utils b/egs2/swbd_sentiment/asr1/utils
new file mode 120000
index 00000000000..f49247da827
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/utils
@@ -0,0 +1 @@
+../../../tools/kaldi/egs/wsj/s5/utils
\ No newline at end of file

From e5c0e0dbdab7e56ea9bf0a852bac10a1d99acf64 Mon Sep 17 00:00:00 2001
From: YushiUeda <yueda@br011.ib.bridges2.psc.edu>
Date: Fri, 4 Mar 2022 10:03:49 -0500
Subject: [PATCH 2/7] add config

---
 egs2/swbd_sentiment/asr1/conf/decode_asr.yaml |   1 +
 egs2/swbd_sentiment/asr1/conf/train_asr.yaml  |   1 +
 .../asr1/conf/tuning/train_asr_conformer.yaml |  62 ++++++++++
 .../tuning/train_asr_conformer_hubert.yaml    |  89 +++++++++++++
 .../tuning/train_asr_conformer_wav2vec2.yaml  |  89 +++++++++++++
 egs2/swbd_sentiment/asr1/local/data.sh        |  10 +-
 .../asr1/local/prepare_sentiment.py           |   4 +-
 egs2/swbd_sentiment/asr1/local/score.py       |   1 +
 egs2/swbd_sentiment/asr1/local/score.sh       | 117 +-----------------
 egs2/swbd_sentiment/asr1/local/score_f1.py    |  86 +++++++++++++
 egs2/swbd_sentiment/asr1/run.sh               |  30 +++++
 11 files changed, 367 insertions(+), 123 deletions(-)
 create mode 100644 egs2/swbd_sentiment/asr1/conf/decode_asr.yaml
 create mode 120000 egs2/swbd_sentiment/asr1/conf/train_asr.yaml
 create mode 100644 egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer.yaml
 create mode 100644 egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer_hubert.yaml
 create mode 100644 egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer_wav2vec2.yaml
 create mode 120000 egs2/swbd_sentiment/asr1/local/score.py
 mode change 100755 => 120000 egs2/swbd_sentiment/asr1/local/score.sh
 create mode 100755 egs2/swbd_sentiment/asr1/local/score_f1.py
 create mode 100755 egs2/swbd_sentiment/asr1/run.sh

diff --git a/egs2/swbd_sentiment/asr1/conf/decode_asr.yaml b/egs2/swbd_sentiment/asr1/conf/decode_asr.yaml
new file mode 100644
index 00000000000..2a0495063be
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/conf/decode_asr.yaml
@@ -0,0 +1 @@
+beam_size: 10
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/conf/train_asr.yaml b/egs2/swbd_sentiment/asr1/conf/train_asr.yaml
new file mode 120000
index 00000000000..10fea4f0971
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/conf/train_asr.yaml
@@ -0,0 +1 @@
+tuning/train_asr_conformer.yaml
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer.yaml b/egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer.yaml
new file mode 100644
index 00000000000..f5104e2d6ea
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer.yaml
@@ -0,0 +1,62 @@
+# network architecture
+# encoder related
+encoder: conformer
+encoder_conf:
+    output_size: 512
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 12
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d
+    normalize_before: true
+    macaron_style: true
+    pos_enc_layer_type: "rel_pos"
+    selfattention_layer_type: "rel_selfattn"
+    activation_type: "swish"
+    use_cnn_module:  true
+    cnn_module_kernel: 31
+# decoder related
+decoder: transformer
+decoder_conf:
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+optim: adam
+optim_conf:
+    lr: 0.0025
+scheduler: warmuplr     # pytorch v1.1.0+ required
+scheduler_conf:
+    warmup_steps: 40000
+batch_type: numel
+batch_bins: 40000000
+accum_grad: 3
+max_epoch: 50
+
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 40
+    num_time_mask: 2
+
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
diff --git a/egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer_hubert.yaml b/egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer_hubert.yaml
new file mode 100644
index 00000000000..e0e9789a3e5
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer_hubert.yaml
@@ -0,0 +1,89 @@
+# network architecture
+# encoder related
+
+encoder: conformer
+encoder_conf:
+  output_size: 512
+  attention_heads: 8
+  linear_units: 2048
+  num_blocks: 12
+  dropout_rate: 0.1
+  positional_dropout_rate: 0.1
+  attention_dropout_rate: 0.1
+  input_layer: conv2d
+  normalize_before: true
+  macaron_style: true
+  pos_enc_layer_type: "rel_pos"
+  selfattention_layer_type: "rel_selfattn"
+  activation_type: "swish"
+  use_cnn_module: true
+  cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+  attention_heads: 8
+  linear_units: 2048
+  num_blocks: 6
+  dropout_rate: 0.1
+  positional_dropout_rate: 0.1
+  self_attention_dropout_rate: 0.1
+  src_attention_dropout_rate: 0.1
+
+optim: adam
+optim_conf:
+  lr: 0.0025
+scheduler: warmuplr   # pytorch v1.1.0+ required #Tune warmup steps
+scheduler_conf:
+  warmup_steps: 25000
+batch_type: numel
+batch_bins: 40000000
+accum_grad: 3
+max_epoch: 50
+
+freeze_param: [
+"frontend.upstream"
+]
+
+frontend_conf:
+ n_fft: 512
+ hop_length: 256
+
+frontend: s3prl
+frontend_conf:
+  frontend_conf:
+    upstream: hubert_large_ll60k # Note: If the upstream is changed, please change the input_size in the preencoder.
+  download_dir: ./hub
+  multilayer_feature: True
+
+preencoder: linear
+preencoder_conf:
+  input_size: 1024 # Note: If the upstream is changed, please change this value accordingly.
+  output_size: 80
+
+model_conf:
+  ctc_weight: 0.3
+  lsm_weight: 0.1
+  length_normalized_loss: false
+  extract_feats_in_collect_stats: false  # Note: "False" means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend.
+
+specaug: specaug
+specaug_conf:
+  apply_time_warp: true
+  time_warp_window: 5
+  time_warp_mode: bicubic
+  apply_freq_mask: true
+  freq_mask_width_range:
+  - 0
+  - 30
+  num_freq_mask: 2
+  apply_time_mask: true
+  time_mask_width_range:
+  - 0
+  - 40
+  num_time_mask: 2
+
+best_model_criterion:
+- - valid
+  - acc
+  - max
+keep_nbest_models: 10
diff --git a/egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer_wav2vec2.yaml b/egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer_wav2vec2.yaml
new file mode 100644
index 00000000000..54c4938f54c
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer_wav2vec2.yaml
@@ -0,0 +1,89 @@
+# network architecture
+# encoder related
+
+encoder: conformer
+encoder_conf:
+  output_size: 512
+  attention_heads: 8
+  linear_units: 2048
+  num_blocks: 12
+  dropout_rate: 0.1
+  positional_dropout_rate: 0.1
+  attention_dropout_rate: 0.1
+  input_layer: conv2d
+  normalize_before: true
+  macaron_style: true
+  pos_enc_layer_type: "rel_pos"
+  selfattention_layer_type: "rel_selfattn"
+  activation_type: "swish"
+  use_cnn_module: true
+  cnn_module_kernel: 31
+
+decoder: transformer
+decoder_conf:
+  attention_heads: 8
+  linear_units: 2048
+  num_blocks: 6
+  dropout_rate: 0.1
+  positional_dropout_rate: 0.1
+  self_attention_dropout_rate: 0.1
+  src_attention_dropout_rate: 0.1
+
+optim: adam
+optim_conf:
+  lr: 0.0025
+scheduler: warmuplr   # pytorch v1.1.0+ required #Tune warmup steps
+scheduler_conf:
+  warmup_steps: 25000
+batch_type: numel
+batch_bins: 40000000
+accum_grad: 3
+max_epoch: 50
+
+freeze_param: [
+"frontend.upstream"
+]
+
+frontend_conf:
+ n_fft: 512
+ hop_length: 256
+
+frontend: s3prl
+frontend_conf:
+  frontend_conf:
+    upstream: wav2vec2_large_ll60k # Note: If the upstream is changed, please change the input_size in the preencoder.
+  download_dir: ./hub
+  multilayer_feature: True
+
+preencoder: linear
+preencoder_conf:
+  input_size: 1024 # Note: If the upstream is changed, please change this value accordingly.
+  output_size: 80
+
+model_conf:
+  ctc_weight: 0.3
+  lsm_weight: 0.1
+  length_normalized_loss: false
+  extract_feats_in_collect_stats: false  # Note: "False" means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend.
+
+specaug: specaug
+specaug_conf:
+  apply_time_warp: true
+  time_warp_window: 5
+  time_warp_mode: bicubic
+  apply_freq_mask: true
+  freq_mask_width_range:
+  - 0
+  - 30
+  num_freq_mask: 2
+  apply_time_mask: true
+  time_mask_width_range:
+  - 0
+  - 40
+  num_time_mask: 2
+
+best_model_criterion:
+- - valid
+  - acc
+  - max
+keep_nbest_models: 10
diff --git a/egs2/swbd_sentiment/asr1/local/data.sh b/egs2/swbd_sentiment/asr1/local/data.sh
index ce85bbf5479..daeb757504e 100755
--- a/egs2/swbd_sentiment/asr1/local/data.sh
+++ b/egs2/swbd_sentiment/asr1/local/data.sh
@@ -58,11 +58,11 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
 fi
 
 if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
-    log " Concatenate Sentiment with Transcript"
-    # concat sentiment (Positive, Negative, Neutral). 
-    # Using annotation reconciliation strategy based on majority voting as in
+    log " Concatenate Sentiment with Transcription"
+    # Concatenate sentiment (Positive, Negative, Neutral) with transcription. 
+    # Using sentiment annotation reconciliation strategy based on majority voting as in
     # https://catalog.ldc.upenn.edu/docs/LDC2020T14/LREC_2020_Switchboard_Senti.pdf
-    # this stage may take a while
+    # This stage may take a while
     mkdir -p data/local/tmp/
     mv -f data/train/* data/local/tmp/.
     mkdir -p data/dev/
@@ -76,7 +76,7 @@ if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
         --wavscp_file data/local/tmp/wav.scp
     for dir in train dev test; do
     utils/utt2spk_to_spk2utt.pl data/$dir/utt2spk > data/$dir/spk2utt
-    utils/validate_data_dir.sh data/$dir
+    utils/fix_data_dir.sh data/$dir
     done
 fi
 
diff --git a/egs2/swbd_sentiment/asr1/local/prepare_sentiment.py b/egs2/swbd_sentiment/asr1/local/prepare_sentiment.py
index 331271efa7d..09a7d51b6b1 100755
--- a/egs2/swbd_sentiment/asr1/local/prepare_sentiment.py
+++ b/egs2/swbd_sentiment/asr1/local/prepare_sentiment.py
@@ -74,7 +74,7 @@ def process_data(target_dir, sentiment_file, text_file, wavscp_file, start_linen
                                 # normalize transcript
                                 transcript = normalize_transcript(transcript)
                                 utt2spk_list.append("{} {}".format(utt_id_tf, spk_id_tf))
-                                segments_list.append("{} {:.2f} {:.2f}".format(utt_id_tf, float(start), float(end)))
+                                segments_list.append("{} {} {:.2f} {:.2f}".format(utt_id_tf, spk_id_tf, float(start), float(end)))
                                 text_list.append("{} {} {}".format(utt_id_tf, label, transcript))
 
                                 if prev_spk_id_tf != spk_id_tf:
@@ -112,7 +112,7 @@ def process_data(target_dir, sentiment_file, text_file, wavscp_file, start_linen
 # Note that there is no "official" split provided. 
 # Using the proportion of train 90%, dev 5%, test 5% as in
 # https://arxiv.org/pdf/1911.09762.pdf
-print("start train file preparation")
+print("start train file preparation...this may take a while")
 process_data(args.train_dir, args.sentiment_file, args.text_file, args.wavscp_file, 0, 47056)
 print("start dev file preparation")
 process_data(args.dev_dir, args.sentiment_file, args.text_file, args.wavscp_file, 47056, 49673)
diff --git a/egs2/swbd_sentiment/asr1/local/score.py b/egs2/swbd_sentiment/asr1/local/score.py
new file mode 120000
index 00000000000..32c1318cf74
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/local/score.py
@@ -0,0 +1 @@
+../../../TEMPLATE/asr1/pyscripts/utils/score_intent.py
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/local/score.sh b/egs2/swbd_sentiment/asr1/local/score.sh
deleted file mode 100755
index a0316ac7c49..00000000000
--- a/egs2/swbd_sentiment/asr1/local/score.sh
+++ /dev/null
@@ -1,116 +0,0 @@
-#!/usr/bin/env bash
-# Copyright Johns Hopkins University (Author: Daniel Povey) 2012.  Apache 2.0.
-
-# begin configuration section.
-cmd=run.pl
-stage=0
-data=data/eval2000
-#end configuration section.
-
-[ -f ./path.sh ] && . ./path.sh
-. parse_options.sh || exit 1;
-
-if [ $# -lt 1 ]; then
-  echo "Usage: local/score.sh [--cmd (run.pl|queue.pl...)] <asr-exp-dir>"
-  echo " Options:"
-  echo "    --cmd (run.pl|queue.pl...)      # specify how to run the sub-processes."
-  echo "    --stage (0|1|2)                 # start scoring script from part-way through."
-  exit 1;
-fi
-
-asr_expdir=$1
-
-hubscr=${KALDI_ROOT}/tools/sctk/bin/hubscr.pl
-[ ! -f ${hubscr} ] && echo "Cannot find scoring program at $hubscr" && exit 1;
-hubdir=$(dirname ${hubscr})
-
-for f in ${data}/stm ${data}/glm; do
-  [ ! -f ${f} ] && echo "$0: expecting file $f to exist" && exit 1;
-done
-name=$(basename ${data}) # e.g. eval2000
-for dir in ${asr_expdir}/decode_*/${name}/score_wer; do
-    score_dir=${dir}/scoring
-    ctm=${score_dir}/hyp.ctm
-    # The WER seems to be lower without converting stm
-    #stm=${score_dir}/ref.stm
-    stm=${data}/stm
-    mkdir -p ${score_dir}
-    if [ ${stage} -le 0 ]; then
-        
-            # Assuming trn files exist
-            #ref=${dir}/ref.wrd.trn
-            hyp=${dir}/hyp.trn
-            # The WER seems to be lower without converting stm
-            #trn2stm.py --orig-stm ${data}/stm ${ref} ${stm}
-            trn2ctm.py ${hyp} ${ctm}
-        
-    fi
-
-    if [ ${stage} -le 1 ]; then
-      # Remove some stuff we don't want to score, from the ctm.
-      # the big expression in parentheses contains all the things that get mapped
-      # by the glm file, into hesitations.
-      # The -$ expression removes partial words.
-      # the aim here is to remove all the things that appear in the reference as optionally
-      # deletable (inside parentheses), as if we delete these there is no loss, while
-      # if we get them correct there is no gain.
-      cp ${ctm} ${score_dir}/tmpf;
-      cat ${score_dir}/tmpf | grep -i -v -E '\[NOISE|LAUGHTER|VOCALIZED-NOISE\]' | \
-      grep -i -v -E '<UNK>' | \
-      grep -i -v -E ' (UH|UM|EH|MM|HM|AH|HUH|HA|ER|OOF|HEE|ACH|EEE|EW)$' | \
-      grep -v -- '-$' > ${ctm};
-    fi
-
-    # Score the set...
-    if [ ${stage} -le 2 ]; then
-        ${cmd} ${score_dir}/score.log ${hubscr} -p ${hubdir} -V -l english -h hub5 -g ${data}/glm -r ${stm} ${ctm} || exit 1;
-    fi
-
-    # For eval2000 score the subsets
-    case "$name" in
-      eval2000*)
-        # Score only the, swbd part...
-        if [ ${stage} -le 3 ]; then
-            swbd_stm=${score_dir}/ref.swbd.stm
-            swbd_ctm=${score_dir}/hyp.swbd.ctm
-            ${cmd} ${score_dir}/score.swbd.log \
-              grep -v '^en_' ${stm} '>' ${swbd_stm} '&&' \
-              grep -v '^en_' ${ctm} '>' ${swbd_ctm} '&&' \
-              ${hubscr} -p ${hubdir} -V -l english -h hub5 -g ${data}/glm -r ${swbd_stm} ${swbd_ctm} || exit 1;
-        fi
-        # Score only the, callhome part...
-        if [ ${stage} -le 3 ]; then
-            callhm_stm=${score_dir}/ref.callhm.stm
-            callhm_ctm=${score_dir}/hyp.callhm.ctm
-            ${cmd} ${score_dir}/score.callhm.log \
-              grep -v '^sw_' ${stm} '>' ${callhm_stm} '&&' \
-              grep -v '^sw_' ${ctm} '>' ${callhm_ctm} '&&' \
-              ${hubscr} -p ${hubdir} -V -l english -h hub5 -g ${data}/glm -r ${callhm_stm} ${callhm_ctm} || exit 1;
-        fi
-        ;;
-    rt03* )
-
-      # Score only the swbd part...
-      if [ ${stage} -le 3 ]; then
-            swbd_stm=${score_dir}/ref.swbd.stm
-            swbd_ctm=${score_dir}/hyp.swbd.ctm
-            ${cmd} ${score_dir}/score.swbd.log \
-              grep -v '^fsh_' ${stm} '>' ${swbd_stm} '&&' \
-              grep -v '^fsh_' ${ctm} '>' ${swbd_ctm} '&&' \
-              ${hubscr} -p ${hubdir} -V -l english -h hub5 -g ${data}/glm -r ${swbd_stm} ${swbd_ctm} || exit 1;
-      fi
-      # Score only the fisher part...
-      if [ ${stage} -le 3 ]; then
-            fsh_stm=${score_dir}/ref.fsh.stm
-            fsh_ctm=${score_dir}/hyp.fsh.ctm
-            ${cmd} ${score_dir}/score.fsh.log \
-              grep -v '^sw_' ${stm} '>' ${fsh_stm} '&&' \
-              grep -v '^sw_' ${ctm} '>' ${fsh_ctm} '&&' \
-              ${hubscr} -p ${hubdir} -V -l english -h hub5 -g ${data}/glm -r ${fsh_stm} ${fsh_ctm} || exit 1;
-      fi
-    ;;
-    esac
-
-    grep 'Percent Total Error' ${score_dir}/hyp.*ctm.filt.dtl
-done
-exit 0
diff --git a/egs2/swbd_sentiment/asr1/local/score.sh b/egs2/swbd_sentiment/asr1/local/score.sh
new file mode 120000
index 00000000000..938c01f1250
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/local/score.sh
@@ -0,0 +1 @@
+../../../slue-voxceleb/asr1/local/score.sh
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/local/score_f1.py b/egs2/swbd_sentiment/asr1/local/score_f1.py
new file mode 100755
index 00000000000..1a44b0abb8d
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/local/score_f1.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+# Copyright 2022  Yushi Ueda
+#           2022  Carnegie Mellon University
+# Apache 2.0
+
+
+import os
+import re
+import sys
+import pandas as pd
+import argparse
+from sklearn.metrics import f1_score
+
+
+def get_classification_result(hyp_file, ref_file):
+    hyp_lines = [line for line in hyp_file]
+    ref_lines = [line for line in ref_file]
+    hyp_list=[]
+    ref_list=[]
+    for line_count in range(len(hyp_lines)):
+        hyp_list.append(hyp_lines[line_count].split(" ")[0])
+        ref_list.append(ref_lines[line_count].split(" ")[0])
+    macro_f1 = f1_score(ref_list, hyp_list, average='macro', labels=["Positive", "Neutral", "Negative"])
+    weighted_f1 = f1_score(ref_list, hyp_list, average='weighted', labels=["Positive", "Neutral", "Negative"])
+    micro_f1 = f1_score(ref_list, hyp_list, average='micro', labels=["Positive", "Neutral", "Negative"])
+    return macro_f1, weighted_f1, micro_f1
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--exp_root", required=True, help="Directory to save experiments")
+parser.add_argument(
+    "--valid_folder",
+    default="inference_asr_model_valid.acc.ave_10best/devel/",
+    help="Directory inside exp_root containing inference on valid set",
+)
+parser.add_argument(
+    "--test_folder",
+    default="inference_asr_model_valid.acc.ave_10best/test/",
+    help="Directory inside exp_root containing inference on test set",
+)
+parser.add_argument(
+    "--utterance_test_folder",
+    default=None,
+    help="Directory inside exp_root containing inference on utterance test set",
+)
+
+args = parser.parse_args()
+
+exp_root = args.exp_root
+valid_inference_folder = args.valid_folder
+test_inference_folder = args.test_folder
+
+valid_hyp_file = open(
+    os.path.join(exp_root, valid_inference_folder + "score_wer/hyp.trn")
+)
+valid_ref_file = open(
+    os.path.join(exp_root, valid_inference_folder + "score_wer/ref.trn")
+)
+
+macro_f1, weighted_f1, micro_f1 = get_classification_result(valid_hyp_file, valid_ref_file)
+print("Valid Intent Classification Result")
+print("macro f1:{}, weighted f1:{}, micro f1:{}".format(macro_f1, weighted_f1, micro_f1))
+
+test_hyp_file = open(
+    os.path.join(exp_root, test_inference_folder + "score_wer/hyp.trn")
+)
+test_ref_file = open(
+    os.path.join(exp_root, test_inference_folder + "score_wer/ref.trn")
+)
+
+macro_f1, weighted_f1, micro_f1 = get_classification_result(test_hyp_file, test_ref_file)
+print("Test Intent Classification Result")
+print("macro f1:{}, weighted f1:{}, micro f1:{}".format(macro_f1, weighted_f1, micro_f1))
+
+if args.utterance_test_folder is not None:
+    utt_test_inference_folder = args.utterance_test_folder
+    utt_test_hyp_file = open(
+        os.path.join(exp_root, utt_test_inference_folder + "score_wer/hyp.trn")
+    )
+    utt_test_ref_file = open(
+        os.path.join(exp_root, utt_test_inference_folder + "score_wer/ref.trn")
+    )
+    macro_f1, weighted_f1, micro_f1 = get_classification_result(utt_test_hyp_file, utt_test_ref_file)
+    print("Unseen Utterance Test Intent Classification Result")
+    print("macro f1:{}, weighted f1:{}, micro f1:{}".format(macro_f1, weighted_f1, micro_f1))
diff --git a/egs2/swbd_sentiment/asr1/run.sh b/egs2/swbd_sentiment/asr1/run.sh
new file mode 100755
index 00000000000..498511133e9
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/run.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+train_set="train"
+valid_set="dev"
+test_sets="test dev"
+
+asr_config=conf/train_asr.yaml
+inference_config=conf/decode_asr.yaml
+
+./asr.sh \
+    --lang en \
+    --ngpu 1 \
+    --use_lm false \
+    --nbpe 5000 \
+    --token_type word\
+    --feats_type raw\
+    --max_wav_duration 30 \
+    --inference_nj 8 \
+    --inference_asr_model valid.acc.ave_10best.pth\
+    --asr_config "${asr_config}" \
+    --inference_config "${inference_config}" \
+    --feats_normalize "utterance_mvn" \
+    --train_set "${train_set}" \
+    --valid_set "${valid_set}" \
+    --test_sets "${test_sets}" "$@"

From 6b27bac6249e78320982173c0664d6cc64ab1e1c Mon Sep 17 00:00:00 2001
From: YushiUeda <yueda@br011.ib.bridges2.psc.edu>
Date: Fri, 4 Mar 2022 16:08:30 -0500
Subject: [PATCH 3/7] update README

---
 egs2/README.md                                |   1 +
 egs2/swbd_sentiment/asr1/README.md            |  33 ++++++
 .../asr1/local/prepare_sentiment.py           | 104 +++++++++++++-----
 3 files changed, 111 insertions(+), 27 deletions(-)
 create mode 100644 egs2/swbd_sentiment/asr1/README.md

diff --git a/egs2/README.md b/egs2/README.md
index d03884f2ceb..dcbd80bf5b9 100755
--- a/egs2/README.md
+++ b/egs2/README.md
@@ -75,6 +75,7 @@ See: https://espnet.github.io/espnet/espnet2_tutorial.html#recipes-using-espnet2
 | su_openslr36            | Sundanese                                                                               | ASR                     | SUN                  | http://www.openslr.org/36                                                                                    |              |
 | swbd                    | Switchboard Corpus for 2-channel Conversational Telephone Speech (300h)                 | ASR                     | ENG                  | https://catalog.ldc.upenn.edu/LDC97S62                                                                       |              |
 | swbd_da                 | NXT Switchboard Annotations                                                             | SLU                     | ENG                  | https://catalog.ldc.upenn.edu/LDC2009T26                                                                     |              |
+| swbd_sentiment          | Speech Sentiment Annotations                                                            | SLU                     | ENG                  | https://catalog.ldc.upenn.edu/LDC2020T14                                                                    |              |
 | tedlium2                | TED-LIUM corpus release 2                                                               | ASR                     | ENG                  | https://www.openslr.org/19/, http://www.lrec-conf.org/proceedings/lrec2014/pdf/1104_Paper.pdf                |              |
 | thchs30                 | A Free Chinese Speech Corpus Released by CSLT@Tsinghua University                       | TTS                     | CMN                  | https://www.openslr.org/18/                                                                                  |              |
 | timit                   | TIMIT Acoustic-Phonetic Continuous Speech Corpus                                        | ASR                     | ENG                  | https://catalog.ldc.upenn.edu/LDC93S1                                                                        |              |
diff --git a/egs2/swbd_sentiment/asr1/README.md b/egs2/swbd_sentiment/asr1/README.md
new file mode 100644
index 00000000000..f486df1e80e
--- /dev/null
+++ b/egs2/swbd_sentiment/asr1/README.md
@@ -0,0 +1,33 @@
+# RESULTS
+## Dataset
+- Speech Sentiment Annotations (Switchboard Sentiment)
+   - Data: https://catalog.ldc.upenn.edu/LDC2020T14
+   - Paper: https://catalog.ldc.upenn.edu/docs/LDC2020T14/LREC_2020_Switchboard_Senti.pdf
+
+## Environments
+- date: `Thu Mar  3 21:34:18 EST 2022`
+- python version: `3.7.11 (default, Jul 27 2021, 14:32:16)  [GCC 7.5.0]`
+- espnet version: `espnet 0.10.7a1`
+- pytorch version: `pytorch 1.9.0+cu102`
+- Git hash: `3b53aedc654fd30a828689c2139a1e130adac077`
+  - Commit date: `Fri Feb 25 00:13:16 2022 -0500`
+
+## Using Conformer based encoder and Transformer based decoder with spectral augmentation and predicting transcript along with sentiment
+- ASR config: [conf/tuning/train_asr_conformer.yaml](conf/tuning/train_asr_conformer.yaml)
+- token_type: word
+- labels: Positive, Neutral, Negative
+
+|dataset|Snt|Intent Classification Macro F1 (%)| Weighted F1 (%)| Micro F1 (%)|
+|---|---|---|---|---|
+|decode_asr_asr_model_valid.acc.ave_10best/valid|2415|61.0|65.0|65.6|
+|decode_asr_asr_model_valid.acc.ave_10best/test|2438|61.4|64.4|64.6|
+
+## Using Conformer based encoder, Transformer based decoder and self-supervised learning features (Wav2vec2.0) with spectral augmentation and predicting transcript along with sentiment
+- ASR config: [conf/tuning/train_asr_conformer_wav2vec2.yaml](conf/tuning/train_asr_conformer_wav2vec2.yaml)
+- token_type: word
+- labels: Positive, Neutral, Negative
+
+|dataset|Snt|Intent Classification Macro F1 (%)| Weighted F1 (%)| Micro F1 (%)|
+|---|---|---|---|---|
+|decode_asr_asr_model_valid.acc.ave_10best/valid|2415|64.5|67.5|67.4|
+|decode_asr_asr_model_valid.acc.ave_10best/test|2438|64.1|66.5|66.3|
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/local/prepare_sentiment.py b/egs2/swbd_sentiment/asr1/local/prepare_sentiment.py
index 09a7d51b6b1..f1c035dc1d1 100755
--- a/egs2/swbd_sentiment/asr1/local/prepare_sentiment.py
+++ b/egs2/swbd_sentiment/asr1/local/prepare_sentiment.py
@@ -3,10 +3,12 @@
 import argparse
 import math
 
+
 def float2str(number, size=6):
     number = str(math.ceil(number * 100))
     return (size - len(number)) * "0" + number
 
+
 def majorityvote(line):
     count_pos = line.count("Positive")
     count_neu = line.count("Neutral")
@@ -18,23 +20,27 @@ def majorityvote(line):
     label = keys[0] if len(keys) == 1 else -1
     return label
 
+
 def normalize_transcript(transcript):
     # remove punctuation except apostrophes
-    transcript = re.sub('(\.|\,|\?|\!|\-|\:|\;)', ' \\1 ', transcript)
-    transcript = re.sub('\.|\,|\?|\!|\-|\:|\;', '', transcript)
+    transcript = re.sub("(\.|\,|\?|\!|\-|\:|\;)", " \\1 ", transcript)
+    transcript = re.sub("\.|\,|\?|\!|\-|\:|\;", "", transcript)
     # remove tag (e.g. [LAUGHTER])
-    transcript = re.sub('\[.+\]', '', transcript)
+    transcript = re.sub("\[.+\]", "", transcript)
     # Detect valid apostrophe cases and split those into two words
-    transcript = re.sub('([a-z])\'([a-z])', '\\1 \'\\2', transcript)
+    transcript = re.sub("([a-z])'([a-z])", "\\1 '\\2", transcript)
     # Clean up special cases of standalone apostrophes
-    transcript = re.sub('([a-z])\' ', '\\1 ', transcript)
+    transcript = re.sub("([a-z])' ", "\\1 ", transcript)
     # remove extra spaces
-    transcript = re.sub(' +', ' ', transcript)
+    transcript = re.sub(" +", " ", transcript)
     # remove space at the beginning of the utterance
-    transcript = re.sub('^ ', '', transcript)
+    transcript = re.sub("^ ", "", transcript)
     return transcript
 
-def process_data(target_dir, sentiment_file, text_file, wavscp_file, start_linenum, end_linenum):
+
+def process_data(
+    target_dir, sentiment_file, text_file, wavscp_file, start_linenum, end_linenum
+):
     if not os.path.exists(target_dir):
         os.makedirs(target_dir)
 
@@ -44,7 +50,9 @@ def process_data(target_dir, sentiment_file, text_file, wavscp_file, start_linen
     wavscp_list = []
     reco2file_list = []
 
-    with open(sentiment_file, "r", encoding="utf-8") as sf, open(text_file, "r", encoding="utf-8") as tf, open(wavscp_file, "r", encoding="utf-8") as wf:
+    with open(sentiment_file, "r", encoding="utf-8") as sf, open(
+        text_file, "r", encoding="utf-8"
+    ) as tf, open(wavscp_file, "r", encoding="utf-8") as wf:
         prev_spk_id_tf = 0
         prev_linenum_tf = 0
         prev_linenum_wf = 0
@@ -69,13 +77,27 @@ def process_data(target_dir, sentiment_file, text_file, wavscp_file, start_linen
                             start_time_id, end_time_id = time_id.split("-")
                             # in case start and end time slightly differ in text and sentiment annotation
                             eps = 0.05
-                            if reco_id_tf == reco_id_sf and start_time_id >= float2str(float(start) - eps) and start_time_id <= float2str(float(start) + eps) and end_time_id >= float2str(float(end) - eps) and end_time_id <= float2str(float(end) + eps):
-                                #print("{} {} {} {}".format(utt_id_tf, reco_id_sf, start_time_id, end_time_id))
+                            if (
+                                reco_id_tf == reco_id_sf
+                                and start_time_id >= float2str(float(start) - eps)
+                                and start_time_id <= float2str(float(start) + eps)
+                                and end_time_id >= float2str(float(end) - eps)
+                                and end_time_id <= float2str(float(end) + eps)
+                            ):
+                                # print("{} {} {} {}".format(utt_id_tf, reco_id_sf, start_time_id, end_time_id))
                                 # normalize transcript
                                 transcript = normalize_transcript(transcript)
-                                utt2spk_list.append("{} {}".format(utt_id_tf, spk_id_tf))
-                                segments_list.append("{} {} {:.2f} {:.2f}".format(utt_id_tf, spk_id_tf, float(start), float(end)))
-                                text_list.append("{} {} {}".format(utt_id_tf, label, transcript))
+                                utt2spk_list.append(
+                                    "{} {}".format(utt_id_tf, spk_id_tf)
+                                )
+                                segments_list.append(
+                                    "{} {} {:.2f} {:.2f}".format(
+                                        utt_id_tf, spk_id_tf, float(start), float(end)
+                                    )
+                                )
+                                text_list.append(
+                                    "{} {} {}".format(utt_id_tf, label, transcript)
+                                )
 
                                 if prev_spk_id_tf != spk_id_tf:
                                     wf.seek(0)
@@ -83,20 +105,42 @@ def process_data(target_dir, sentiment_file, text_file, wavscp_file, start_linen
                                         if linenum_wf >= prev_linenum_wf:
                                             spk_id_wf = line_wf.split(" ")[0]
                                             if spk_id_wf == spk_id_tf:
-                                                wavscp_list.append("{}".format(line_wf.strip("\n")))
-                                                reco_id_wf, channel_id = spk_id_wf.split("-")
-                                                reco2file_list.append("{} {} {}".format(spk_id_wf, reco_id_wf, channel_id))
+                                                wavscp_list.append(
+                                                    "{}".format(line_wf.strip("\n"))
+                                                )
+                                                (
+                                                    reco_id_wf,
+                                                    channel_id,
+                                                ) = spk_id_wf.split("-")
+                                                reco2file_list.append(
+                                                    "{} {} {}".format(
+                                                        spk_id_wf,
+                                                        reco_id_wf,
+                                                        channel_id,
+                                                    )
+                                                )
                                                 prev_linenum_wf = linenum_wf
                                                 break
                                 prev_spk_id_tf = spk_id_tf
                                 prev_linenum_tf = linenum_tf
                                 break
-    with open(os.path.join(target_dir,"utt2spk"), "w", encoding="utf-8") as utt2spk, open(os.path.join(target_dir,"segments"), "w", encoding="utf-8") as segments, open(os.path.join(target_dir,"text"), "w", encoding="utf-8") as text, open(os.path.join(target_dir,"wav.scp"), "w", encoding="utf-8") as wavscp, open(os.path.join(target_dir,"reco2file_and_channel"), "w", encoding="utf-8") as reco2file:
-        utt2spk.write("\n".join(utt2spk_list)+"\n")
-        segments.write("\n".join(segments_list)+"\n")
-        text.write("\n".join(text_list)+"\n")
-        wavscp.write("\n".join(wavscp_list)+"\n")
-        reco2file.write("\n".join(reco2file_list)+"\n")
+    with open(
+        os.path.join(target_dir, "utt2spk"), "w", encoding="utf-8"
+    ) as utt2spk, open(
+        os.path.join(target_dir, "segments"), "w", encoding="utf-8"
+    ) as segments, open(
+        os.path.join(target_dir, "text"), "w", encoding="utf-8"
+    ) as text, open(
+        os.path.join(target_dir, "wav.scp"), "w", encoding="utf-8"
+    ) as wavscp, open(
+        os.path.join(target_dir, "reco2file_and_channel"), "w", encoding="utf-8"
+    ) as reco2file:
+        utt2spk.write("\n".join(utt2spk_list) + "\n")
+        segments.write("\n".join(segments_list) + "\n")
+        text.write("\n".join(text_list) + "\n")
+        wavscp.write("\n".join(wavscp_list) + "\n")
+        reco2file.write("\n".join(reco2file_list) + "\n")
+
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--train_dir", type=str, default="data/train")
@@ -109,14 +153,20 @@ def process_data(target_dir, sentiment_file, text_file, wavscp_file, start_linen
 args = parser.parse_args()
 
 # Split into train, dev, test
-# Note that there is no "official" split provided. 
+# Note that there is no "official" split provided.
 # Using the proportion of train 90%, dev 5%, test 5% as in
 # https://arxiv.org/pdf/1911.09762.pdf
 print("start train file preparation...this may take a while")
-process_data(args.train_dir, args.sentiment_file, args.text_file, args.wavscp_file, 0, 47056)
+process_data(
+    args.train_dir, args.sentiment_file, args.text_file, args.wavscp_file, 0, 47056
+)
 print("start dev file preparation")
-process_data(args.dev_dir, args.sentiment_file, args.text_file, args.wavscp_file, 47056, 49673)
+process_data(
+    args.dev_dir, args.sentiment_file, args.text_file, args.wavscp_file, 47056, 49673
+)
 print("start test file preparation")
-process_data(args.test_dir, args.sentiment_file, args.text_file, args.wavscp_file, 49673, 52293)
+process_data(
+    args.test_dir, args.sentiment_file, args.text_file, args.wavscp_file, 49673, 52293
+)
 
 print("Successfully finished text, utt2spk, segments, wavescp preparation")

From 328f051db9c8ec92bf85a7f91db5acae49dd33f7 Mon Sep 17 00:00:00 2001
From: YushiUeda <yueda@br011.ib.bridges2.psc.edu>
Date: Fri, 4 Mar 2022 16:10:16 -0500
Subject: [PATCH 4/7] update README

---
 egs2/swbd_sentiment/asr1/README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/egs2/swbd_sentiment/asr1/README.md b/egs2/swbd_sentiment/asr1/README.md
index f486df1e80e..84ee7efbbf1 100644
--- a/egs2/swbd_sentiment/asr1/README.md
+++ b/egs2/swbd_sentiment/asr1/README.md
@@ -16,6 +16,7 @@
 - ASR config: [conf/tuning/train_asr_conformer.yaml](conf/tuning/train_asr_conformer.yaml)
 - token_type: word
 - labels: Positive, Neutral, Negative
+- Pre-trained Model: https://huggingface.co/espnet/YushiUeda_swbd_sentiment_asr_train_asr_conformer
 
 |dataset|Snt|Intent Classification Macro F1 (%)| Weighted F1 (%)| Micro F1 (%)|
 |---|---|---|---|---|
@@ -26,6 +27,7 @@
 - ASR config: [conf/tuning/train_asr_conformer_wav2vec2.yaml](conf/tuning/train_asr_conformer_wav2vec2.yaml)
 - token_type: word
 - labels: Positive, Neutral, Negative
+- Pre-trained Model: https://huggingface.co/espnet/YushiUeda_swbd_sentiment_asr_train_asr_conformer_wav2vec2
 
 |dataset|Snt|Intent Classification Macro F1 (%)| Weighted F1 (%)| Micro F1 (%)|
 |---|---|---|---|---|

From a68f7838e7a81b15222e07ed65c51f14ca8a2f61 Mon Sep 17 00:00:00 2001
From: YushiUeda <yueda@br011.ib.bridges2.psc.edu>
Date: Fri, 4 Mar 2022 16:15:20 -0500
Subject: [PATCH 5/7] delete comment

---
 egs2/swbd_sentiment/asr1/local/data.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/egs2/swbd_sentiment/asr1/local/data.sh b/egs2/swbd_sentiment/asr1/local/data.sh
index daeb757504e..54df2f8eb97 100755
--- a/egs2/swbd_sentiment/asr1/local/data.sh
+++ b/egs2/swbd_sentiment/asr1/local/data.sh
@@ -22,9 +22,6 @@ log "$0 $*"
 . ./path.sh
 . ./cmd.sh
 
-# for local usage. Delete when making a PR.
-SWBD="swbd"
-
 if [ $# -ne 0 ]; then
     log "Error: No positional arguments are required."
     exit 2

From 1b31f46ab681a421b0372f2ba24a685af2911536 Mon Sep 17 00:00:00 2001
From: YushiUeda <yueda@br011.ib.bridges2.psc.edu>
Date: Fri, 4 Mar 2022 16:42:49 -0500
Subject: [PATCH 6/7] apply black

---
 egs2/swbd_sentiment/asr1/local/score_f1.py | 45 ++++++++++++++++------
 1 file changed, 34 insertions(+), 11 deletions(-)

diff --git a/egs2/swbd_sentiment/asr1/local/score_f1.py b/egs2/swbd_sentiment/asr1/local/score_f1.py
index 1a44b0abb8d..a36c37c7b1f 100755
--- a/egs2/swbd_sentiment/asr1/local/score_f1.py
+++ b/egs2/swbd_sentiment/asr1/local/score_f1.py
@@ -16,14 +16,23 @@
 def get_classification_result(hyp_file, ref_file):
     hyp_lines = [line for line in hyp_file]
     ref_lines = [line for line in ref_file]
-    hyp_list=[]
-    ref_list=[]
+    hyp_list = []
+    ref_list = []
     for line_count in range(len(hyp_lines)):
         hyp_list.append(hyp_lines[line_count].split(" ")[0])
         ref_list.append(ref_lines[line_count].split(" ")[0])
-    macro_f1 = f1_score(ref_list, hyp_list, average='macro', labels=["Positive", "Neutral", "Negative"])
-    weighted_f1 = f1_score(ref_list, hyp_list, average='weighted', labels=["Positive", "Neutral", "Negative"])
-    micro_f1 = f1_score(ref_list, hyp_list, average='micro', labels=["Positive", "Neutral", "Negative"])
+    macro_f1 = f1_score(
+        ref_list, hyp_list, average="macro", labels=["Positive", "Neutral", "Negative"]
+    )
+    weighted_f1 = f1_score(
+        ref_list,
+        hyp_list,
+        average="weighted",
+        labels=["Positive", "Neutral", "Negative"],
+    )
+    micro_f1 = f1_score(
+        ref_list, hyp_list, average="micro", labels=["Positive", "Neutral", "Negative"]
+    )
     return macro_f1, weighted_f1, micro_f1
 
 
@@ -58,9 +67,13 @@ def get_classification_result(hyp_file, ref_file):
     os.path.join(exp_root, valid_inference_folder + "score_wer/ref.trn")
 )
 
-macro_f1, weighted_f1, micro_f1 = get_classification_result(valid_hyp_file, valid_ref_file)
+macro_f1, weighted_f1, micro_f1 = get_classification_result(
+    valid_hyp_file, valid_ref_file
+)
 print("Valid Intent Classification Result")
-print("macro f1:{}, weighted f1:{}, micro f1:{}".format(macro_f1, weighted_f1, micro_f1))
+print(
+    "macro f1:{}, weighted f1:{}, micro f1:{}".format(macro_f1, weighted_f1, micro_f1)
+)
 
 test_hyp_file = open(
     os.path.join(exp_root, test_inference_folder + "score_wer/hyp.trn")
@@ -69,9 +82,13 @@ def get_classification_result(hyp_file, ref_file):
     os.path.join(exp_root, test_inference_folder + "score_wer/ref.trn")
 )
 
-macro_f1, weighted_f1, micro_f1 = get_classification_result(test_hyp_file, test_ref_file)
+macro_f1, weighted_f1, micro_f1 = get_classification_result(
+    test_hyp_file, test_ref_file
+)
 print("Test Intent Classification Result")
-print("macro f1:{}, weighted f1:{}, micro f1:{}".format(macro_f1, weighted_f1, micro_f1))
+print(
+    "macro f1:{}, weighted f1:{}, micro f1:{}".format(macro_f1, weighted_f1, micro_f1)
+)
 
 if args.utterance_test_folder is not None:
     utt_test_inference_folder = args.utterance_test_folder
@@ -81,6 +98,12 @@ def get_classification_result(hyp_file, ref_file):
     utt_test_ref_file = open(
         os.path.join(exp_root, utt_test_inference_folder + "score_wer/ref.trn")
     )
-    macro_f1, weighted_f1, micro_f1 = get_classification_result(utt_test_hyp_file, utt_test_ref_file)
+    macro_f1, weighted_f1, micro_f1 = get_classification_result(
+        utt_test_hyp_file, utt_test_ref_file
+    )
     print("Unseen Utterance Test Intent Classification Result")
-    print("macro f1:{}, weighted f1:{}, micro f1:{}".format(macro_f1, weighted_f1, micro_f1))
+    print(
+        "macro f1:{}, weighted f1:{}, micro f1:{}".format(
+            macro_f1, weighted_f1, micro_f1
+        )
+    )

From d537330b2503ead5d0b70e82387161c8624825e6 Mon Sep 17 00:00:00 2001
From: YushiUeda <yueda@br013.ib.bridges2.psc.edu>
Date: Sun, 6 Mar 2022 16:28:40 -0500
Subject: [PATCH 7/7] modify config & fix ci error

---
 egs2/swbd_sentiment/asr1/conf/decode_asr.yaml |  2 +-
 .../tuning/train_asr_conformer_hubert.yaml    | 89 -------------------
 .../tuning/train_asr_conformer_wav2vec2.yaml  |  1 +
 .../asr1/local/prepare_sentiment.py           | 16 ++--
 4 files changed, 11 insertions(+), 97 deletions(-)
 delete mode 100644 egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer_hubert.yaml

diff --git a/egs2/swbd_sentiment/asr1/conf/decode_asr.yaml b/egs2/swbd_sentiment/asr1/conf/decode_asr.yaml
index 2a0495063be..e2408c35e82 100644
--- a/egs2/swbd_sentiment/asr1/conf/decode_asr.yaml
+++ b/egs2/swbd_sentiment/asr1/conf/decode_asr.yaml
@@ -1 +1 @@
-beam_size: 10
\ No newline at end of file
+beam_size: 1
\ No newline at end of file
diff --git a/egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer_hubert.yaml b/egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer_hubert.yaml
deleted file mode 100644
index e0e9789a3e5..00000000000
--- a/egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer_hubert.yaml
+++ /dev/null
@@ -1,89 +0,0 @@
-# network architecture
-# encoder related
-
-encoder: conformer
-encoder_conf:
-  output_size: 512
-  attention_heads: 8
-  linear_units: 2048
-  num_blocks: 12
-  dropout_rate: 0.1
-  positional_dropout_rate: 0.1
-  attention_dropout_rate: 0.1
-  input_layer: conv2d
-  normalize_before: true
-  macaron_style: true
-  pos_enc_layer_type: "rel_pos"
-  selfattention_layer_type: "rel_selfattn"
-  activation_type: "swish"
-  use_cnn_module: true
-  cnn_module_kernel: 31
-
-decoder: transformer
-decoder_conf:
-  attention_heads: 8
-  linear_units: 2048
-  num_blocks: 6
-  dropout_rate: 0.1
-  positional_dropout_rate: 0.1
-  self_attention_dropout_rate: 0.1
-  src_attention_dropout_rate: 0.1
-
-optim: adam
-optim_conf:
-  lr: 0.0025
-scheduler: warmuplr   # pytorch v1.1.0+ required #Tune warmup steps
-scheduler_conf:
-  warmup_steps: 25000
-batch_type: numel
-batch_bins: 40000000
-accum_grad: 3
-max_epoch: 50
-
-freeze_param: [
-"frontend.upstream"
-]
-
-frontend_conf:
- n_fft: 512
- hop_length: 256
-
-frontend: s3prl
-frontend_conf:
-  frontend_conf:
-    upstream: hubert_large_ll60k # Note: If the upstream is changed, please change the input_size in the preencoder.
-  download_dir: ./hub
-  multilayer_feature: True
-
-preencoder: linear
-preencoder_conf:
-  input_size: 1024 # Note: If the upstream is changed, please change this value accordingly.
-  output_size: 80
-
-model_conf:
-  ctc_weight: 0.3
-  lsm_weight: 0.1
-  length_normalized_loss: false
-  extract_feats_in_collect_stats: false  # Note: "False" means during collect stats (stage 10), generating dummy stats files rather than extract_feats by forward frontend.
-
-specaug: specaug
-specaug_conf:
-  apply_time_warp: true
-  time_warp_window: 5
-  time_warp_mode: bicubic
-  apply_freq_mask: true
-  freq_mask_width_range:
-  - 0
-  - 30
-  num_freq_mask: 2
-  apply_time_mask: true
-  time_mask_width_range:
-  - 0
-  - 40
-  num_time_mask: 2
-
-best_model_criterion:
-- - valid
-  - acc
-  - max
-keep_nbest_models: 10
diff --git a/egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer_wav2vec2.yaml b/egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer_wav2vec2.yaml
index 54c4938f54c..92b01329911 100644
--- a/egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer_wav2vec2.yaml
+++ b/egs2/swbd_sentiment/asr1/conf/tuning/train_asr_conformer_wav2vec2.yaml
@@ -52,6 +52,7 @@ frontend: s3prl
 frontend_conf:
   frontend_conf:
     upstream: wav2vec2_large_ll60k # Note: If the upstream is changed, please change the input_size in the preencoder.
+    # If using hubert, change the above line to "upstream: hubert_large_ll60k"
   download_dir: ./hub
   multilayer_feature: True
 
diff --git a/egs2/swbd_sentiment/asr1/local/prepare_sentiment.py b/egs2/swbd_sentiment/asr1/local/prepare_sentiment.py
index f1c035dc1d1..8921fa4272d 100755
--- a/egs2/swbd_sentiment/asr1/local/prepare_sentiment.py
+++ b/egs2/swbd_sentiment/asr1/local/prepare_sentiment.py
@@ -23,10 +23,10 @@ def majorityvote(line):
 
 def normalize_transcript(transcript):
     # remove punctuation except apostrophes
-    transcript = re.sub("(\.|\,|\?|\!|\-|\:|\;)", " \\1 ", transcript)
-    transcript = re.sub("\.|\,|\?|\!|\-|\:|\;", "", transcript)
+    transcript = re.sub(r"(\.|\,|\?|\!|\-|\:|\;)", " \\1 ", transcript)
+    transcript = re.sub(r"\.|\,|\?|\!|\-|\:|\;", "", transcript)
     # remove tag (e.g. [LAUGHTER])
-    transcript = re.sub("\[.+\]", "", transcript)
+    transcript = re.sub(r"\[.+\]", "", transcript)
     # Detect valid apostrophe cases and split those into two words
     transcript = re.sub("([a-z])'([a-z])", "\\1 '\\2", transcript)
     # Clean up special cases of standalone apostrophes
@@ -58,7 +58,8 @@ def process_data(
         prev_linenum_wf = 0
         for linenum, line_sf in enumerate(sf):
             if linenum >= start_linenum and linenum < end_linenum:
-                # "sw02005_0[tab]0.0[tab]11.287375[tab]Neutral-{Questioning}#Neutral-{No emotion}#Neutral-{No emotion}"
+                # "sw02005_0[tab]0.0[tab]11.287375[tab]
+                # Neutral-{Questioning}#Neutral-{No emotion}#Neutral-{No emotion}"
                 utt_id_sf, start, end, sentiment = line_sf.strip().split("\t")
                 # "sw02005_0" -> "sw02005"
                 reco_id_sf = utt_id_sf.split("_")[0]
@@ -67,7 +68,8 @@ def process_data(
                     tf.seek(0)
                     for linenum_tf, line_tf in enumerate(tf):
                         if linenum_tf >= prev_linenum_tf:
-                            # "sw02001-A_018732-018950 oh i see uh-huh" -> "sw02001-A_018732-018950" "oh i see uh-huh"
+                            # "sw02001-A_018732-018950 oh i see uh-huh"
+                            # -> "sw02001-A_018732-018950" "oh i see uh-huh"
                             utt_id_tf, transcript = line_tf.strip("\n").split(" ", 1)
                             # "sw02001-A_018732-018950" -> "sw02001-A" "018732-018950"
                             spk_id_tf, time_id = utt_id_tf.split("_")
@@ -75,7 +77,8 @@ def process_data(
                             reco_id_tf = spk_id_tf.split("-")[0]
                             # "018732-018950" -> "018732" "018950"
                             start_time_id, end_time_id = time_id.split("-")
-                            # in case start and end time slightly differ in text and sentiment annotation
+                            # in case start and end time slightly differ
+                            # in text and sentiment annotation
                             eps = 0.05
                             if (
                                 reco_id_tf == reco_id_sf
@@ -84,7 +87,6 @@ def process_data(
                                 and end_time_id >= float2str(float(end) - eps)
                                 and end_time_id <= float2str(float(end) + eps)
                             ):
-                                # print("{} {} {} {}".format(utt_id_tf, reco_id_sf, start_time_id, end_time_id))
                                 # normalize transcript
                                 transcript = normalize_transcript(transcript)
                                 utt2spk_list.append(