Fix bug about add_special_tokens and so on (huggingface#31496)

* fix bug about add_special_tokens and so on

* improve add_special_tokens and padding behavior

* add a test case for add_special_tokens and padding

src/transformers/pipelines/text_generation.py

@@ -137,11 +137,10 @@ def _sanitize_parameters(
 
         add_special_tokens = False
         if "add_special_tokens" in generate_kwargs:
-            preprocess_params["add_special_tokens"] = generate_kwargs["add_special_tokens"]
-            add_special_tokens = generate_kwargs["add_special_tokens"]
+            add_special_tokens = preprocess_params["add_special_tokens"] = generate_kwargs.pop("add_special_tokens")
 
         if "padding" in generate_kwargs:
-            preprocess_params["padding"] = generate_kwargs["padding"]
+            preprocess_params["padding"] = generate_kwargs.pop("padding")
 
         if truncation is not None:
             preprocess_params["truncation"] = truncation

tests/pipelines/test_pipelines_text_generation.py

@@ -107,6 +107,20 @@ def test_small_model_pt(self):
         )
         assert output_str != output_str_with_truncation  # results must be different because one had truncation
 
+        ## -- test kwargs for preprocess_params
+        outputs = text_generator("This is a test", do_sample=False, add_special_tokens=False, padding=False)
+        self.assertEqual(
+            outputs,
+            [
+                {
+                    "generated_text": (
+                        "This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope."
+                        " oscope. FiliFili@@"
+                    )
+                }
+            ],
+        )
+
         # -- what is the point of this test? padding is hardcoded False in the pipeline anyway
         text_generator.tokenizer.pad_token_id = text_generator.model.config.eos_token_id
         text_generator.tokenizer.pad_token = "<pad>"