fix: incorrect role assigning in rewardmodelfilter

camel/datagen/self_instruct/filter/filter_function.py

@@ -172,7 +172,12 @@ class RewardModelFilter(FilterFunction):
             to pass the filter.
     """
 
-    def __init__(self, reward_model: BaseRewardModel, threshold: float = 0.5):
+    def __init__(
+        self,
+        reward_model: BaseRewardModel,
+        threshold: float = 0.5,
+    ):
+        self.prompt = ""
         self.reward_model = reward_model
         self.threshold = threshold
 
@@ -190,8 +195,11 @@ def apply(self, instruction: str) -> bool:
                 required score is not found in `scores`.
         """
 
-        messages = [{"role": "user", "content": instruction}]
-        scores = self.reward_model.evaluate(messages)
+        data = [
+            {"role": "user", "content": self.prompt},
+            {"role": "assistant", "content": instruction},
+        ]
+        scores = self.reward_model.evaluate(data)
         score_types = self.reward_model.get_scores_types()
         if not score_types:
             raise ValueError("No score types available from the reward model.")

camel/datagen/self_instruct/filter/instruction_filter.py

@@ -13,7 +13,7 @@
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 from typing import Any, Dict, List
 
-from .filter_function import FilterFunction
+from .filter_function import FilterFunction, RewardModelFilter
 from .filter_registry import FILTER_REGISTRY
 
 
@@ -53,10 +53,13 @@ def add_filter(self, filter_function: FilterFunction):
         """
         self.filters.append(filter_function)
 
-    def filter(self, instruction: str, return_details: bool = False):
+    def filter(
+        self, prompt: str, instruction: str, return_details: bool = False
+    ):
         r"""Check if the given instruction passes all filter functions.
 
         Args:
+            prompt (str): The prompt of generating the instruction.
             instruction (str): The instruction to evaluate.
             return_details (bool): If True, returns a tuple (bool, List[str])
                 where the list contains the names of filters that failed.
@@ -68,6 +71,8 @@ def filter(self, instruction: str, return_details: bool = False):
         """
         failed_filters = []
         for f in self.filters:
+            if isinstance(f, RewardModelFilter):
+                f.prompt = prompt
             if not f.apply(instruction):
                 failed_filters.append(type(f).__name__)
 

camel/datagen/self_instruct/self_instruct.py

@@ -136,14 +136,14 @@ def sample_machine_tasks(self, count: int) -> List[dict]:
 
         return random.sample(self.machine_tasks, count)
 
-    def generate_machine_instruction(self) -> str:
+    def generate_machine_instruction(self) -> List:
         r"""Generate a machine instruction using the agent.
 
         Combines human and machine tasks based on the configured ratio to
             create a prompt for instruction generation.
 
         Returns:
-            str: A machine-generated instruction.
+            List: The prompt and a machine-generated instruction.
         """
 
         sampled_human_tasks = self.sample_human_tasks(
@@ -176,7 +176,7 @@ def generate_machine_instruction(self) -> str:
             for line in response.msgs[0].content.split("\n")
             if line.strip()
         ]
-        return generated_tasks[0]
+        return [prompt, generated_tasks[0]]
 
     def identify_instruction(self, instruction: str) -> bool:
         r"""Determine if the given instruction is a classification task.
@@ -371,8 +371,8 @@ def generate(self):
             for f in self.instruction_filter.filters:
                 if isinstance(f, RougeSimilarityFilter):
                     f.existing_instructions = existing_instructions
-            instruction = self.generate_machine_instruction()
-            if self.instruction_filter.filter(instruction):
+            prompt, instruction = self.generate_machine_instruction()
+            if self.instruction_filter.filter(prompt, instruction):
                 instruction_dict = {
                     "id": f"machine_task_{len(self.machine_tasks) + 1}",
                     "instruction": instruction,