Fix add_generation_prompt arg for paged transformers in GRPO and RLOO trainers (#4370)

Co-authored-by: Quentin Gallouédec <gallouedec.quentin@gmail.com>

Author: albertvillanova

trl/trainer/grpo_trainer.py

@@ -1263,12 +1263,15 @@ def _generate_single_turn(self, prompts: list):
             processor_kwargs = {
                 "max_length": self.max_prompt_length,
                 "truncation": True,
-                "add_generation_prompt": True,
                 "add_special_tokens": False,
             }
             if is_conversational({"prompt": prompts[0]}):
                 processor_outputs = self.processing_class.apply_chat_template(
-                    conversation=prompts, **processor_kwargs, tokenize=True, return_dict=True
+                    conversation=prompts,
+                    **processor_kwargs,
+                    add_generation_prompt=True,
+                    tokenize=True,
+                    return_dict=True,
                 )
             else:
                 processor_outputs = self.processing_class(text=prompts, **processor_kwargs)

trl/trainer/rloo_trainer.py

@@ -1088,13 +1088,13 @@ def _generate_single_turn(self, prompts: list):
             processor_kwargs = {
                 "max_length": self.max_prompt_length,
                 "truncation": True,
-                "add_generation_prompt": True,
                 "add_special_tokens": False,
             }
             if is_conversational({"prompt": prompts[0]}):
                 processor_outputs = self.processing_class.apply_chat_template(
                     conversation=prompts,
                     **processor_kwargs,
+                    add_generation_prompt=True,
                     tokenize=True,
                     return_dict=True,
                 )
@@ -1137,7 +1137,7 @@ def _generate_single_turn(self, prompts: list):
                 generate_inputs = self.processing_class.apply_chat_template(
                     conversation=prompts,
                     **processor_kwargs,
-                    add_generation_kwargs=True,
+                    add_generation_prompt=True,
                     tokenize=True,
                     return_dict=True,
                 )