Use save_jinja_files instead of save_raw_chat_template_files

Author: Rocketknight1

src/transformers/processing_utils.py

@@ -642,7 +642,7 @@ def save_pretrained(self, save_directory, push_to_hub: bool = False, **kwargs):
         processor_dict = self.to_dict()
         # Save `chat_template` in its own file. We can't get it from `processor_dict` as we popped it in `to_dict`
         # to avoid serializing chat template in json config file. So let's get it from `self` directly
-        save_as_jinja = kwargs.get("save_raw_chat_template", False)
+        save_as_jinja = kwargs.get("save_jinja_files", False)
         is_single_template = isinstance(self.chat_template, str)
 
         if save_as_jinja and is_single_template:
@@ -677,7 +677,7 @@ def save_pretrained(self, save_directory, push_to_hub: bool = False, **kwargs):
             # chat template dicts are saved to chat_template.json as lists of dicts with fixed key names.
             raise ValueError(
                 "Multiple chat templates are not supported in the legacy format. Please save them as separate files "
-                "using the `save_raw_chat_template` argument."
+                "using the `save_jinja_files` argument."
             )
 
         # For now, let's not save to `processor_config.json` if the processor doesn't have extra attributes and

src/transformers/tokenization_utils_base.py

@@ -2487,15 +2487,15 @@ def save_pretrained(
             tokenizer_config.update(self.extra_special_tokens)
 
         saved_raw_chat_template_files = []
-        if kwargs.get("save_raw_chat_template", False) and isinstance(self.chat_template, str):
+        if kwargs.get("save_jinja_files", False) and isinstance(self.chat_template, str):
             # New format for single templates is to save them as chat_template.jinja
             with open(chat_template_file, "w", encoding="utf-8") as f:
                 f.write(self.chat_template)
             logger.info(f"chat template saved in {chat_template_file}")
             saved_raw_chat_template_files.append(chat_template_file)
             if "chat_template" in tokenizer_config:
                 tokenizer_config.pop("chat_template")  # To ensure it doesn't somehow end up in the config too
-        elif kwargs.get("save_raw_chat_template", False) and isinstance(self.chat_template, dict):
+        elif kwargs.get("save_jinja_files", False) and isinstance(self.chat_template, dict):
             # New format for multiple templates is to save the default as chat_template.jinja
             # and the other templates in the chat_templates/ directory
             for template_name, template in self.chat_template.items():

src/transformers/utils/hub.py

@@ -962,9 +962,9 @@ def push_to_hub(
         """
         use_auth_token = deprecated_kwargs.pop("use_auth_token", None)
         ignore_metadata_errors = deprecated_kwargs.pop("ignore_metadata_errors", False)
-        save_raw_chat_template = deprecated_kwargs.pop(
-            "save_raw_chat_template", None
-        )  # TODO: This is only used for testing and should be removed once save_raw_chat_template becomes the default
+        save_jinja_files = deprecated_kwargs.pop(
+            "save_jinja_files", None
+        )  # TODO: This is only used for testing and should be removed once save_jinja_files becomes the default
         if use_auth_token is not None:
             warnings.warn(
                 "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",
@@ -1021,12 +1021,12 @@ def push_to_hub(
             files_timestamps = self._get_files_timestamps(work_dir)
 
             # Save all files.
-            if save_raw_chat_template:
+            if save_jinja_files:
                 self.save_pretrained(
                     work_dir,
                     max_shard_size=max_shard_size,
                     safe_serialization=safe_serialization,
-                    save_raw_chat_template=True,
+                    save_jinja_files=True,
                 )
             else:
                 self.save_pretrained(work_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)

tests/test_processing_common.py

@@ -748,7 +748,7 @@ def test_chat_template_save_loading(self):
             self.assertEqual(getattr(reloaded_processor.tokenizer, "chat_template", None), existing_tokenizer_template)
 
         with tempfile.TemporaryDirectory() as tmpdirname:
-            processor.save_pretrained(tmpdirname, save_raw_chat_template=True)
+            processor.save_pretrained(tmpdirname, save_jinja_files=True)
             self.assertTrue(Path(tmpdirname, "chat_template.jinja").is_file())
             self.assertFalse(Path(tmpdirname, "chat_template.json").is_file())
             self.assertFalse(Path(tmpdirname, "additional_chat_templates").is_dir())
@@ -760,7 +760,7 @@ def test_chat_template_save_loading(self):
 
         with tempfile.TemporaryDirectory() as tmpdirname:
             processor.chat_template = {"default": "a", "secondary": "b"}
-            processor.save_pretrained(tmpdirname, save_raw_chat_template=True)
+            processor.save_pretrained(tmpdirname, save_jinja_files=True)
             self.assertTrue(Path(tmpdirname, "chat_template.jinja").is_file())
             self.assertFalse(Path(tmpdirname, "chat_template.json").is_file())
             self.assertTrue(Path(tmpdirname, "additional_chat_templates").is_dir())
@@ -772,7 +772,7 @@ def test_chat_template_save_loading(self):
 
         with tempfile.TemporaryDirectory() as tmpdirname:
             processor.chat_template = {"default": "a", "secondary": "b"}
-            processor.save_pretrained(tmpdirname, save_raw_chat_template=False)
+            processor.save_pretrained(tmpdirname, save_jinja_files=False)
             self.assertFalse(Path(tmpdirname, "chat_template.jinja").is_file())
             self.assertTrue(Path(tmpdirname, "chat_template.json").is_file())
             self.assertFalse(Path(tmpdirname, "additional_chat_templates").is_dir())

tests/test_tokenization_common.py

@@ -1164,7 +1164,7 @@ def test_chat_template(self):
                 new_tokenizer.apply_chat_template(dummy_conversation, tokenize=True, return_dict=False)
 
                 with tempfile.TemporaryDirectory() as tmp_dir_name:
-                    save_files = tokenizer.save_pretrained(tmp_dir_name, save_raw_chat_template=True)
+                    save_files = tokenizer.save_pretrained(tmp_dir_name, save_jinja_files=True)
                     # Check we are saving a chat_template.jinja file
                     self.assertTrue(any(file.endswith("chat_template.jinja") for file in save_files))
                     chat_template_file = Path(tmp_dir_name) / "chat_template.jinja"
@@ -1190,7 +1190,7 @@ def test_chat_template_save_loading(self):
                 self.skipTest("tokenizer doesn't accept chat templates at input")
             tokenizer.chat_template = "test template"
             with tempfile.TemporaryDirectory() as tmpdirname:
-                tokenizer.save_pretrained(tmpdirname, save_raw_chat_template=True)
+                tokenizer.save_pretrained(tmpdirname, save_jinja_files=True)
                 self.assertTrue(Path(tmpdirname, "chat_template.jinja").is_file())
                 self.assertFalse(Path(tmpdirname, "chat_template.json").is_file())
                 self.assertFalse(Path(tmpdirname, "additional_chat_templates").is_dir())
@@ -1202,7 +1202,7 @@ def test_chat_template_save_loading(self):
 
             with tempfile.TemporaryDirectory() as tmpdirname:
                 tokenizer.chat_template = {"default": "a", "secondary": "b"}
-                tokenizer.save_pretrained(tmpdirname, save_raw_chat_template=True)
+                tokenizer.save_pretrained(tmpdirname, save_jinja_files=True)
                 self.assertTrue(Path(tmpdirname, "chat_template.jinja").is_file())
                 self.assertFalse(Path(tmpdirname, "chat_template.json").is_file())
                 self.assertTrue(Path(tmpdirname, "additional_chat_templates").is_dir())
@@ -1214,7 +1214,7 @@ def test_chat_template_save_loading(self):
 
             with tempfile.TemporaryDirectory() as tmpdirname:
                 tokenizer.chat_template = {"default": "a", "secondary": "b"}
-                tokenizer.save_pretrained(tmpdirname, save_raw_chat_template=False)
+                tokenizer.save_pretrained(tmpdirname, save_jinja_files=False)
                 self.assertFalse(Path(tmpdirname, "chat_template.jinja").is_file())
                 self.assertFalse(Path(tmpdirname, "chat_template.json").is_file())
                 self.assertFalse(Path(tmpdirname, "additional_chat_templates").is_dir())
@@ -1713,12 +1713,12 @@ def test_chat_template_dict_saving(self):
         tokenizers = self.get_tokenizers()
         for tokenizer in tokenizers:
             with self.subTest(f"{tokenizer.__class__.__name__}"):
-                for save_raw_chat_template in (True, False):
+                for save_jinja_files in (True, False):
                     tokenizer.chat_template = {"default": dummy_template_1, "template2": dummy_template_2}
                     with tempfile.TemporaryDirectory() as tmp_dir_name:
-                        # Test that save_raw_chat_template is ignored when there's a dict of multiple templates
-                        tokenizer.save_pretrained(tmp_dir_name, save_raw_chat_template=save_raw_chat_template)
-                        if save_raw_chat_template:
+                        # Test that save_jinja_files is ignored when there's a dict of multiple templates
+                        tokenizer.save_pretrained(tmp_dir_name, save_jinja_files=save_jinja_files)
+                        if save_jinja_files:
                             config_dict = json.load(open(os.path.join(tmp_dir_name, "tokenizer_config.json")))
                             self.assertNotIn("chat_template", config_dict)
                             self.assertTrue(os.path.exists(os.path.join(tmp_dir_name, "chat_template.jinja")))
@@ -1749,7 +1749,7 @@ def test_chat_template_file_priority(self):
             with self.subTest(f"{tokenizer.__class__.__name__}"):
                 with tempfile.TemporaryDirectory() as tmp_dir_name:
                     tokenizer.chat_template = dummy_template1
-                    tokenizer.save_pretrained(tmp_dir_name, save_raw_chat_template=False)
+                    tokenizer.save_pretrained(tmp_dir_name, save_jinja_files=False)
                     with Path(tmp_dir_name, "chat_template.jinja").open("w") as f:
                         f.write(dummy_template2)
                     new_tokenizer = tokenizer.from_pretrained(tmp_dir_name)