Skip to content

Commit 12cc313

Browse files
committed
Reorder to fix errors
1 parent 7df1482 commit 12cc313

File tree

1 file changed

+18
-18
lines changed

1 file changed

+18
-18
lines changed

src/transformers/tokenization_utils_base.py

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1985,24 +1985,6 @@ def from_pretrained(
19851985
"chat_template_file": CHAT_TEMPLATE_FILE,
19861986
}
19871987

1988-
# This block looks for any extra chat template files
1989-
if is_local:
1990-
template_dir = Path(pretrained_model_name_or_path, CHAT_TEMPLATE_DIR)
1991-
if template_dir.is_dir():
1992-
for template_file in template_dir.glob("*.jinja"):
1993-
template_name = template_file.name.removesuffix(".jinja")
1994-
additional_files_names[f"chat_template_{template_name}"] = (
1995-
f"{CHAT_TEMPLATE_DIR}/{template_file.name}"
1996-
)
1997-
else:
1998-
for template in list_repo_templates(
1999-
pretrained_model_name_or_path,
2000-
local_files_only=local_files_only,
2001-
revision=revision,
2002-
cache_dir=cache_dir,
2003-
):
2004-
additional_files_names[f"chat_template_{template}"] = f"{CHAT_TEMPLATE_DIR}/{template}.jinja"
2005-
20061988
vocab_files = {**cls.vocab_files_names, **additional_files_names}
20071989
if "tokenizer_file" in vocab_files:
20081990
# Try to get the tokenizer config to see if there are versioned tokenizer files.
@@ -2032,6 +2014,24 @@ def from_pretrained(
20322014
fast_tokenizer_file = get_fast_tokenizer_file(tokenizer_config["fast_tokenizer_files"])
20332015
vocab_files["tokenizer_file"] = fast_tokenizer_file
20342016

2017+
# This block looks for any extra chat template files
2018+
if is_local:
2019+
template_dir = Path(pretrained_model_name_or_path, CHAT_TEMPLATE_DIR)
2020+
if template_dir.is_dir():
2021+
for template_file in template_dir.glob("*.jinja"):
2022+
template_name = template_file.name.removesuffix(".jinja")
2023+
vocab_files[f"chat_template_{template_name}"] = (
2024+
f"{CHAT_TEMPLATE_DIR}/{template_file.name}"
2025+
)
2026+
else:
2027+
for template in list_repo_templates(
2028+
pretrained_model_name_or_path,
2029+
local_files_only=local_files_only,
2030+
revision=revision,
2031+
cache_dir=cache_dir,
2032+
):
2033+
vocab_files[f"chat_template_{template}"] = f"{CHAT_TEMPLATE_DIR}/{template}.jinja"
2034+
20352035
# Get files from url, cache, or disk depending on the case
20362036
resolved_vocab_files = {}
20372037
for file_id, file_path in vocab_files.items():

0 commit comments

Comments
 (0)