Update preprocess_commonvoice.py

k2-fsa · Mar 13, 2024 · 09a358a · 09a358a
1 parent a39aa8a
commit 09a358a
Showing 1 changed file with 17 additions and 2 deletions.
diff --git a/egs/commonvoice/ASR/local/preprocess_commonvoice.py b/egs/commonvoice/ASR/local/preprocess_commonvoice.py
@@ -56,15 +56,30 @@ def normalize_text(utt: str, language: str) -> str:
         # Mozilla Common Voice uses both "yue" and "zh-HK" for Cantonese
         # Not sure why they decided to do this...
         return (
-            utt.replace(" ", "")
-            .replace("，", "")
+            utt.replace("，", "")
             .replace("。", " ")
             .replace("？", "")
             .replace("！", "")
             .replace("?", "")
             .replace("!", "")
             .replace("‘", "")
             .replace("、", "")
+            .replace(",", "")
+            .replace(".", "")
+            .replace(":", "")
+            .replace(";", "")
+            .replace("「", "")
+            .replace("」", "")
+            .replace("“", "")
+            .replace("”", "")
+            .replace("\\", "")
+            .replace("~", "")
+            .replace("—", "")
+            .replace("ㄧ", "")
+            .replace("《", "")
+            .replace("》", "")
+            .replace("…", "")
+            .replace("⋯", "")
             .upper()
         )
     else: