[Doc] Drop dummy reward and dataset for DeepMath-103K and accuracy reward (#4524)

Author: qgallouedec

README.md

@@ -92,16 +92,13 @@ trainer.train()
 ```python
 from datasets import load_dataset
 from trl import GRPOTrainer
+from trl.rewards import accuracy_reward
 
-dataset = load_dataset("trl-lib/tldr", split="train")
-
-# Dummy reward function: count the number of unique characters in the completions
-def reward_num_unique_chars(completions, **kwargs):
-    return [len(set(c)) for c in completions]
+dataset = load_dataset("trl-lib/DeepMath-103K", split="train")
 
 trainer = GRPOTrainer(
     model="Qwen/Qwen2-0.5B-Instruct",
-    reward_funcs=reward_num_unique_chars,
+    reward_funcs=accuracy_reward,
     train_dataset=dataset,
 )
 trainer.train()

docs/source/grpo_trainer.md

@@ -14,10 +14,10 @@ This post-training method was contributed by [Quentin Gallouédec](https://huggi
 
 ## Quick start
 
-This example demonstrates how to train a model using the GRPO method. We train a [Qwen 0.5B Instruct model](https://huggingface.co/Qwen/Qwen2-0.5B-Instruct) with the prompts from the [UltraFeedback prompts dataset](https://huggingface.co/datasets/trl-lib/ultrafeedback-prompt). You can view the data in the dataset here:
+This example demonstrates how to train a model using the GRPO method. We train a [Qwen 0.5B Instruct model](https://huggingface.co/Qwen/Qwen2-0.5B-Instruct) with the prompts from the [DeepMath-103K dataset](https://huggingface.co/datasets/trl-lib/DeepMath-103K). You can view the data in the dataset here:
 
 <iframe
-  src="https://huggingface.co/datasets/trl-lib/ultrafeedback-prompt/embed/viewer/default/train?row=0"
+  src="https://huggingface.co/datasets/trl-lib/DeepMath-103K/embed/viewer/default/train?row=0"
   frameborder="0"
   width="100%"
   height="560px"
@@ -28,21 +28,14 @@ Below is the script to train the model.
 ```python
 # train_grpo.py
 from datasets import load_dataset
-from trl import GRPOConfig, GRPOTrainer
-
-dataset = load_dataset("trl-lib/ultrafeedback-prompt", split="train")
+from trl import GRPOTrainer
+from trl.rewards import accuracy_reward
 
-# Dummy reward function for demonstration purposes
-def reward_num_unique_letters(completions, **kwargs):
-    """Reward function that rewards completions with more unique letters."""
-    completion_contents = [completion[0]["content"] for completion in completions]
-    return [float(len(set(content))) for content in completion_contents]
+dataset = load_dataset("trl-lib/DeepMath-103K", split="train")
 
-training_args = GRPOConfig(output_dir="Qwen2-0.5B-GRPO")
 trainer = GRPOTrainer(
     model="Qwen/Qwen2-0.5B-Instruct",
-    reward_funcs=reward_num_unique_letters,
-    args=training_args,
+    reward_funcs=accuracy_reward,
     train_dataset=dataset,
 )
 trainer.train()
@@ -290,29 +283,27 @@ import argparse
 
 from datasets import load_dataset
 from trl import GRPOTrainer, GRPOConfig
+from trl.rewards import accuracy_reward
 
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("--vllm_server_host", type=str, default="", help="The server IP")
     args = parser.parse_args()
 
-    # Example dataset from TLDR
-    dataset = load_dataset("trl-lib/tldr", split="train")
-
-    # Dummy reward function: count the number of unique characters in the completions
-    def reward_num_unique_chars(completions, **kwargs):
-        return [len(set(c)) for c in completions]
+    dataset = load_dataset("trl-lib/DeepMath-103K", split="train")
 
     training_args = GRPOConfig(
-        output_dir="Qwen2.5-72B-GRPO",
         per_device_train_batch_size=4,
-        bf16=True,
-        gradient_checkpointing=True,
         use_vllm=True,
         vllm_server_host=args.vllm_server_host.replace("ip-", "").replace("-", "."),  # from ip-X-X-X-X to X.X.X.X
     )
 
-    trainer = GRPOTrainer(model="Qwen/Qwen2.5-72B", args=training_args, reward_funcs=reward_num_unique_chars, train_dataset=dataset)
+    trainer = GRPOTrainer(
+        model="Qwen/Qwen2.5-72B",
+        args=training_args,
+        reward_funcs=accuracy_reward,
+        train_dataset=dataset
+    )
     trainer.train()
 
 if __name__=="__main__":

docs/source/quickstart.md

@@ -24,15 +24,12 @@ trainer.train()
 ```python
 from trl import GRPOTrainer
 from datasets import load_dataset
-
-# Define a simple reward function (count unique chars as example)
-def reward_function(completions, **kwargs):
-    return [len(set(completion.lower())) for completion in completions]
+from trl.rewards import accuracy_reward
 
 trainer = GRPOTrainer(
     model="Qwen/Qwen2.5-0.5B-Instruct",  # Start from SFT model
-    train_dataset=load_dataset("trl-lib/tldr", split="train"),
-    reward_funcs=reward_function,
+    train_dataset=load_dataset("trl-lib/DeepMath-103K", split="train"),
+    reward_funcs=accuracy_reward,
 )
 trainer.train()
 ```

docs/source/rloo_trainer.md

@@ -15,10 +15,10 @@ This post-training method was contributed by [Costa Huang](https://github.com/vw
 
 ## Quick start
 
-This example demonstrates how to train a model using the RLOO method. We train a [Qwen 0.5B Instruct model](https://huggingface.co/Qwen/Qwen2-0.5B-Instruct) with the prompts from the [UltraFeedback prompts dataset](https://huggingface.co/datasets/trl-lib/ultrafeedback-prompt). You can view the data in the dataset here:
+This example demonstrates how to train a model using the RLOO method. We train a [Qwen 0.5B Instruct model](https://huggingface.co/Qwen/Qwen2-0.5B-Instruct) with the prompts from the [DeepMath-103K dataset](https://huggingface.co/datasets/trl-lib/DeepMath-103K). You can view the data in the dataset here:
 
 <iframe
-  src="https://huggingface.co/datasets/trl-lib/ultrafeedback-prompt/embed/viewer/default/train?row=0"
+  src="https://huggingface.co/datasets/trl-lib/DeepMath-103K/embed/viewer/default/train?row=0"
   frameborder="0"
   width="100%"
   height="560px"
@@ -29,21 +29,14 @@ Below is the script to train the model.
 ```python
 # train_rloo.py
 from datasets import load_dataset
-from trl import RLOOConfig, RLOOTrainer
-
-dataset = load_dataset("trl-lib/ultrafeedback-prompt", split="train")
+from trl import RLOOTrainer
+from trl.rewards import accuracy_reward
 
-# Dummy reward function for demonstration purposes
-def reward_num_unique_letters(completions, **kwargs):
-    """Reward function that rewards completions with more unique letters."""
-    completion_contents = [completion[0]["content"] for completion in completions]
-    return [float(len(set(content))) for content in completion_contents]
+dataset = load_dataset("trl-lib/DeepMath-103K", split="train")
 
-training_args = RLOOConfig(output_dir="Qwen2-0.5B-RLOO")
 trainer = RLOOTrainer(
     model="Qwen/Qwen2-0.5B-Instruct",
-    reward_funcs=reward_num_unique_letters,
-    args=training_args,
+    reward_funcs=accuracy_reward,
     train_dataset=dataset,
 )
 trainer.train()

docs/source/vllm_integration.md

@@ -46,24 +46,14 @@ Sample of a simple `train.py` script:
 ```python
 from datasets import load_dataset
 from trl import GRPOTrainer, GRPOConfig
+from trl.rewards import accuracy_reward
 
-dataset = load_dataset("trl-lib/tldr", split="train")
-
-# Dummy reward function: count the number of unique characters in the completions
-def reward_num_unique_chars(completions, **kwargs):
-    return [len(set(c)) for c in completions]
-
-training_args = GRPOConfig(
-    output_dir="my_test",
-    use_vllm=True,
-    bf16=True,
-    gradient_checkpointing=True,
-)
+dataset = load_dataset("trl-lib/DeepMath-103K", split="train")
 
 trainer = GRPOTrainer(
     model="Qwen/Qwen2.5-7B",
-    args=training_args,
-    reward_funcs=reward_num_unique_chars,
+    args=GRPOConfig(use_vllm=True),
+    reward_funcs=accuracy_reward,
     train_dataset=dataset,
 )
 
@@ -76,24 +66,14 @@ trainer.train()
 ```python
 from datasets import load_dataset
 from trl import OnlineDPOTrainer, OnlineDPOConfig
+from trl.rewards import accuracy_reward
 
-dataset = load_dataset("trl-lib/tldr", split="train")
-
-# Dummy reward function: count the number of unique characters in the completions
-def reward_num_unique_chars(completions, **kwargs):
-    return [len(set(c)) for c in completions]
-
-training_args = OnlineDPOConfig(
-    output_dir="my_test",
-    use_vllm=True,
-    bf16=True,
-    gradient_checkpointing=True,
-)
+dataset = load_dataset("trl-lib/DeepMath-103K", split="train")
 
 trainer = OnlineDPOTrainer(
     model="Qwen/Qwen2.5-7B",
-    args=training_args,
-    reward_funcs=reward_num_unique_chars,
+    args=OnlineDPOConfig(use_vllm=True),
+    reward_funcs=accuracy_reward,
     train_dataset=dataset,
 )
 
@@ -106,24 +86,14 @@ trainer.train()
 ```python
 from datasets import load_dataset
 from trl.experimental.nash_md import NashMDConfig, NashMDTrainer
+from trl.rewards import accuracy_reward
 
-dataset = load_dataset("trl-lib/tldr", split="train")
-
-# Dummy reward function: count the number of unique characters in the completions
-def reward_num_unique_chars(completions, **kwargs):
-    return [len(set(c)) for c in completions]
-
-training_args = NashMDConfig(
-    output_dir="my_test",
-    use_vllm=True,
-    bf16=True,
-    gradient_checkpointing=True,
-)
+dataset = load_dataset("trl-lib/DeepMath-103K", split="train")
 
 trainer = NashMDTrainer(
     model="Qwen/Qwen2.5-7B",
-    args=training_args,
-    reward_funcs=reward_num_unique_chars,
+    args=NashMDConfig(use_vllm=True),
+    reward_funcs=accuracy_reward,
     train_dataset=dataset,
 )
 
@@ -135,25 +105,15 @@ trainer.train()
 
 ```python
 from datasets import load_dataset
-from trl.experimental.xpo import XPOTrainer, XPOConfig
-
-dataset = load_dataset("trl-lib/tldr", split="train")
+from trl import XPOTrainer, XPOConfig
+from trl.rewards import accuracy_reward
 
-# Dummy reward function: count the number of unique characters in the completions
-def reward_num_unique_chars(completions, **kwargs):
-    return [len(set(c)) for c in completions]
-
-training_args = XPOConfig(
-    output_dir="my_test",
-    use_vllm=True,
-    bf16=True,
-    gradient_checkpointing=True,
-)
+dataset = load_dataset("trl-lib/DeepMath-103K", split="train")
 
 trainer = XPOTrainer(
     model="Qwen/Qwen2.5-7B",
-    args=training_args,
-    reward_funcs=reward_num_unique_chars,
+    args=XPOConfig(use_vllm=True),
+    reward_funcs=accuracy_reward,
     train_dataset=dataset,
 )
 
@@ -166,24 +126,14 @@ trainer.train()
 ```python
 from datasets import load_dataset
 from trl import RLOOTrainer, RLOOConfig
+from trl.rewards import accuracy_reward
 
-dataset = load_dataset("trl-lib/tldr", split="train")
-
-# Dummy reward function: count the number of unique characters in the completions
-def reward_num_unique_chars(completions, **kwargs):
-    return [len(set(c)) for c in completions]
-
-training_args = RLOOConfig(
-    output_dir="my_test",
-    use_vllm=True,
-    bf16=True,
-    gradient_checkpointing=True,
-)
+dataset = load_dataset("trl-lib/DeepMath-103K", split="train")
 
 trainer = RLOOTrainer(
     model="Qwen/Qwen2.5-7B",
-    args=training_args,
-    reward_funcs=reward_num_unique_chars,
+    args=RLOOConfig(use_vllm=True),
+    reward_funcs=accuracy_reward,
     train_dataset=dataset,
 )