Merge pull request huggingface#8 from huggingface/convert-with-eos

pcuenca · web-flow · commit d9b65411a5be · 2025-03-11T12:50:59.000+01:00
Add eos tokens for instruct models
diff --git a/src/transformers/models/gemma3/convert_gemma3_weights_orbax_to_hf.py b/src/transformers/models/gemma3/convert_gemma3_weights_orbax_to_hf.py
@@ -509,8 +509,9 @@ def main(*args):
     )
 
     if INCLUDE_CHAT_TEMPLATE.value:
-        # Include chat temaplate for CausalLM models
+        # Include chat template for CausalLM models
         tokenizer.chat_template = _CHAT_TEMPLATE
+        config.eos_token_id = [1, 106]
 
     if _TEXT_ONLY.value:
         config.vision_config = None

Original file line number	Diff line number	Diff line change
`@@ -509,8 +509,9 @@ def main(*args):`
`509`	`509`	`)`
`510`	`510`
`511`	`511`	`if INCLUDE_CHAT_TEMPLATE.value:`
`512`		`- # Include chat temaplate for CausalLM models`
	`512`	`+ # Include chat template for CausalLM models`
`513`	`513`	`tokenizer.chat_template = _CHAT_TEMPLATE`
	`514`	`+ config.eos_token_id = [1, 106]`
`514`	`515`
`515`	`516`	`if _TEXT_ONLY.value:`
`516`	`517`	`config.vision_config = None`