Render chat template tojson filter as unicode (#31041)

* Render chat template tojson filter as unicode * ruff--
huggingface · May 28, 2024 · 22dab24 · 22dab24
1 parent 4f98b14
commit 22dab24
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py
@@ -1853,6 +1853,7 @@ def raise_exception(message):
  raise TemplateError(message)
 
  jinja_env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True)
+ jinja_env.policies["json.dumps_kwargs"]["ensure_ascii"] = False
  jinja_env.globals["raise_exception"] = raise_exception
  return jinja_env.from_string(chat_template)