adding pad token to fix huggingface#28534

JINO-ROHIT · Apr 10, 2024 · 5e68f6a · 5e68f6a
1 parent 1f57dc7
commit 5e68f6a
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/examples/pytorch/text-classification/run_glue_no_trainer.py b/examples/pytorch/text-classification/run_glue_no_trainer.py
@@ -327,7 +327,8 @@ def main():
     tokenizer = AutoTokenizer.from_pretrained(
         args.model_name_or_path, use_fast=not args.use_slow_tokenizer, trust_remote_code=args.trust_remote_code
     )
-    tokenizer.pad_token = tokenizer.eos_token
+    if tokenizer.pad_token is None: 
+        tokenizer.pad_token = tokenizer.eos_token
     config.pad_token_id = tokenizer.pad_token_id
 
     model = AutoModelForSequenceClassification.from_pretrained(