load metric with evaluate (EleutherAI#2351)

somosnlp · Oct 9, 2024 · d26f89d · d26f89d
1 parent cc5f8ea
commit d26f89d
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/lm_eval/tasks/squadv2/task.py b/lm_eval/tasks/squadv2/task.py
@@ -37,7 +37,9 @@
 
 
 def _squad_metric(predictions, references):
-    squad_metric = datasets.load_metric("squad_v2")
+    import evaluate
+
+    squad_metric = evaluate.load("squad_v2")
     return squad_metric.compute(predictions=predictions, references=references)