Make GPQA CoT metric pattern matching less strict (#3420)

stanford-crfm · Mar 10, 2025 · b1bc37e · b1bc37e
1 parent 84d37d7
commit b1bc37e
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 4 deletions.
diff --git a/...chmark/metrics/chain_of_thought_metric.py → ...k/metrics/gpqa_chain_of_thought_metric.py b/...chmark/metrics/chain_of_thought_metric.py → ...k/metrics/gpqa_chain_of_thought_metric.py
@@ -30,11 +30,16 @@ def extract_answer(output_text: str) -> Optional[str]:
     if match:
         return match.group(1)
 
-    # If neither regex matches, return None
+    # Third regex: Matches "answer is (A-J)" with optional leading non-alpha characters
+    match = re.search(r"correct answer is [^A-Za-z]*([A-J])", output_text)
+    if match:
+        return match.group(1)
+
+    # If no regex matches, return None
     return None
 
 
-class ChainOfThoughtMetric(Metric):
+class GPQAChainOfThoughtMetric(Metric):
     """
     This metric focuses on structured reasoning and the accuracy of extracted answers.
     It compares model outputs against correct answers provided in a multiple-choice

diff --git a/src/helm/benchmark/run_specs/capabilities_run_specs.py b/src/helm/benchmark/run_specs/capabilities_run_specs.py
@@ -58,10 +58,13 @@ def get_mmlu_pro_spec(subject: str, use_chain_of_thought: str = "true", use_few_
             adapter_spec=adapter_spec,
             metric_specs=get_basic_metric_specs([])
             + [
-                MetricSpec(class_name="helm.benchmark.metrics.chain_of_thought_metric.ChainOfThoughtMetric", args={}),
+                MetricSpec(
+                    class_name="helm.benchmark.metrics.gpqa_chain_of_thought_metric.GPQAChainOfThoughtMetric", args={}
+                ),
             ],
             groups=["mmlu_pro"],
         )
+
     else:
         adapter_spec = AdapterSpec(
             method=ADAPT_MULTIPLE_CHOICE_JOINT,
@@ -167,7 +170,9 @@ def get_gpqa_spec(subset: str, use_chain_of_thought: str = "true", use_few_shot:
         (
             get_basic_metric_specs([])
             + [
-                MetricSpec(class_name="helm.benchmark.metrics.chain_of_thought_metric.ChainOfThoughtMetric", args={}),
+                MetricSpec(
+                    class_name="helm.benchmark.metrics.gpqa_chain_of_thought_metric.GPQAChainOfThoughtMetric", args={}
+                ),
             ]
         )
         if use_chain_of_thought_bool