From 2cea058123a61ad4002c937b3172240930eaa88d Mon Sep 17 00:00:00 2001 From: "jun.4" Date: Wed, 29 May 2024 14:32:14 +0900 Subject: [PATCH 1/3] Change JSON serialization to custom json.dumps to prevent escaping of "<", ">", "&", "'" --- src/transformers/tokenization_utils_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 116fbfdf7bbbf0..1a1532c6c2362c 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1853,7 +1853,7 @@ def raise_exception(message): raise TemplateError(message) jinja_env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True) - jinja_env.policies["json.dumps_kwargs"]["ensure_ascii"] = False + jinja_env.filters["tojson"] = lambda x: json.dumps(x, ensure_ascii=False, sort_keys=True) jinja_env.globals["raise_exception"] = raise_exception return jinja_env.from_string(chat_template) From 409c89676250ccf41aa67f235f175ed0c7dafba0 Mon Sep 17 00:00:00 2001 From: "jun.4" Date: Fri, 31 May 2024 15:37:27 +0900 Subject: [PATCH 2/3] caller has control over the order, remove sort_key=True --- src/transformers/tokenization_utils_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 1a1532c6c2362c..a4c5267bcf37f8 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1853,7 +1853,7 @@ def raise_exception(message): raise TemplateError(message) jinja_env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True) - jinja_env.filters["tojson"] = lambda x: json.dumps(x, ensure_ascii=False, sort_keys=True) + jinja_env.filters["tojson"] = lambda x: json.dumps(x, ensure_ascii=False) jinja_env.globals["raise_exception"] = raise_exception return jinja_env.from_string(chat_template) From b5d2ba0614fdf112661aefbbdcd3ec651a712939 Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 12 Jun 2024 14:34:44 +0100 Subject: [PATCH 3/3] Move tojson into a proper function and expose a couple of other args --- src/transformers/tokenization_utils_base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index a4c5267bcf37f8..bc9fc19eaa0e23 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1852,8 +1852,13 @@ def _compile_jinja_template(self, chat_template): def raise_exception(message): raise TemplateError(message) + def tojson(x, ensure_ascii=False, indent=None, separators=None, sort_keys=False): + # We override the built-in tojson filter because Jinja's default filter escapes HTML characters + # We also expose some options like custom indents and separators + return json.dumps(x, ensure_ascii=ensure_ascii, indent=indent, separators=separators, sort_keys=sort_keys) + jinja_env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True) - jinja_env.filters["tojson"] = lambda x: json.dumps(x, ensure_ascii=False) + jinja_env.filters["tojson"] = tojson jinja_env.globals["raise_exception"] = raise_exception return jinja_env.from_string(chat_template)