Fix GLUE MNLI when using max_eval_samples (huggingface#18722)

oneraghavan · Sep 21, 2022 · ef6741f · ef6741f
1 parent 18643ff
commit ef6741f
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/examples/pytorch/text-classification/run_glue.py b/examples/pytorch/text-classification/run_glue.py
@@ -549,7 +549,11 @@ def compute_metrics(p: EvalPrediction):
         eval_datasets = [eval_dataset]
         if data_args.task_name == "mnli":
             tasks.append("mnli-mm")
-            eval_datasets.append(raw_datasets["validation_mismatched"])
+            valid_mm_dataset = raw_datasets["validation_mismatched"]
+            if data_args.max_eval_samples is not None:
+                max_eval_samples = min(len(valid_mm_dataset), data_args.max_eval_samples)
+                valid_mm_dataset = valid_mm_dataset.select(range(max_eval_samples))
+            eval_datasets.append(valid_mm_dataset)
             combined = {}
 
         for eval_dataset, task in zip(eval_datasets, tasks):