Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix Chinese lang from 'zh' to 'zh-cn' in xtts tokenizer #3242

Closed
wants to merge 1 commit into from
Closed

fix Chinese lang from 'zh' to 'zh-cn' in xtts tokenizer #3242

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions TTS/tts/layers/xtts/tokenizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@


def get_spacy_lang(lang):
if lang == "zh":
if lang in ["zh", "zh-cn"]:
return Chinese()
elif lang == "ja":
return Japanese()
Expand Down Expand Up @@ -170,7 +170,7 @@ def split_sentence(text, lang, text_split_length=250):
# There are not many common abbreviations in Arabic as in English.
]
],
"zh": [
"zh-cn": [
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
for x in [
# Chinese doesn't typically use abbreviations in the same way as Latin-based scripts.
Expand Down Expand Up @@ -335,7 +335,7 @@ def expand_abbreviations_multilingual(text, lang="en"):
("°", " درجة "),
]
],
"zh": [
"zh-cn": [
# Chinese
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
for x in [
Expand Down Expand Up @@ -519,7 +519,7 @@ def _expand_number(m, lang="en"):


def expand_numbers_multilingual(text, lang="en"):
if lang == "zh":
if lang in ["zh", "zh-cn"]:
text = zh_num2words()(text)
else:
if lang in ["en", "ru"]:
Expand Down Expand Up @@ -602,6 +602,7 @@ def __init__(self, vocab_file=None):
"pt": 203,
"pl": 224,
"zh": 82,
"zh-cn": 82,
"ar": 166,
"cs": 186,
"ru": 182,
Expand All @@ -627,9 +628,9 @@ def check_input_length(self, txt, lang):
)

def preprocess_text(self, txt, lang):
if lang in {"ar", "cs", "de", "en", "es", "fr", "hu", "it", "nl", "pl", "pt", "ru", "tr", "zh", "ko"}:
if lang in {"ar", "cs", "de", "en", "es", "fr", "hu", "it", "nl", "pl", "pt", "ru", "tr", "zh", "zh-cn", "ko"}:
txt = multilingual_cleaners(txt, lang)
if lang == "zh":
if lang == "zh" or lang == "zh-cn":
txt = chinese_transliterate(txt)
if lang == "ko":
txt = korean_transliterate(txt)
Expand Down
Loading