Update preprocess_commonvoice.py

k2-fsa · Mar 13, 2024 · eaceb69 · eaceb69
1 parent b30a4d6
commit eaceb69
Showing 1 changed file with 12 additions and 1 deletion.
diff --git a/egs/commonvoice/ASR/local/preprocess_commonvoice.py b/egs/commonvoice/ASR/local/preprocess_commonvoice.py
@@ -21,7 +21,7 @@
 from pathlib import Path
 from typing import Optional
 
-from lhotse import CutSet, SupervisionSegment
+from lhotse import CutSet
 from lhotse.recipes.utils import read_manifests_if_cached
 
 
@@ -82,6 +82,17 @@ def normalize_text(utt: str, language: str) -> str:
             .replace("…", "")
             .replace("⋯", "")
             .replace("·", "")
+            .replace("﹒", "")
+            .replace("．", "")
+            .replace("：", "")
+            .replace("︰", "")
+            .replace("﹖", "")
+            .replace("（", "")
+            .replace("）", "")
+            .replace("－", "")
+            .replace("～", "")
+            .replace("；", "")
+            .replace("", "")
             .upper()
         )
     else: