mcscript

guides/tasks/supported_tasks.md

@@ -23,6 +23,7 @@
 | EP-DPR | dpr | ✅ |  | dpr | Edge-Probing |
 | GLUE Diagnostic | glue_diagnostics | ✅ | ✅ | glue_diagnostics | GLUE |
 | HellaSwag | hellaswag | ✅ | ✅ | hellaswag |  |
+| [MCScript2.0](https://arxiv.org/pdf/1905.09531.pdf) | mcscript | ✅ |  | mcscript | [Data](https://my.hidrive.com/share/wdnind8pp5#$/) |
 | MLM | * | ✅ | * | mlm_simple | See task-specific notes. |
 | MLQA | `mlqa_{lang1}_{lang2}` | ✅ | ✅ | mlqa | XTREME, multi-lang |
 | MNLI | mnli | ✅ | ✅ | mnli | GLUE, MNLI-matched |

jiant/tasks/evaluate/core.py

@@ -929,6 +929,7 @@ def get_evaluation_scheme_for_task(task) -> BaseEvaluationScheme:
             tasks.WnliTask,
             tasks.WSCTask,
             tasks.XnliTask,
+            tasks.MCScriptTask,
             tasks.ArctTask,
         ),
     ):

jiant/tasks/lib/mcscript.py

@@ -0,0 +1,70 @@
+from dataclasses import dataclass
+
+from jiant.tasks.lib.templates.shared import labels_to_bimap
+from jiant.tasks.lib.templates import multiple_choice as mc_template
+from jiant.utils.python.io import read_json_lines
+
+
+@dataclass
+class Example(mc_template.Example):
+    pass
+
+
+@dataclass
+class TokenizedExample(mc_template.TokenizedExample):
+    pass
+
+
+@dataclass
+class DataRow(mc_template.DataRow):
+    pass
+
+
+@dataclass
+class Batch(mc_template.Batch):
+    pass
+
+
+class MCScriptTask(mc_template.AbstractMultipleChoiceTask):
+    Example = Example
+    TokenizedExample = Example
+    DataRow = DataRow
+    Batch = Batch
+
+    CHOICE_KEYS = [0, 1]
+    CHOICE_TO_ID, ID_TO_CHOICE = labels_to_bimap(CHOICE_KEYS)
+    NUM_CHOICES = len(CHOICE_KEYS)
+
+    def get_train_examples(self):
+        return self._create_examples(lines=read_json_lines(self.train_path), set_type="train")
+
+    def get_val_examples(self):
+        return self._create_examples(lines=read_json_lines(self.val_path), set_type="val")
+
+    def get_test_examples(self):
+        return self._create_examples(lines=read_json_lines(self.test_path), set_type="test")
+
+    @classmethod
+    def _create_examples(cls, lines, set_type):
+        examples = []
+        for line in lines:
+            passage = line["passage"]["text"]
+            passage_id = line["idx"]
+            for question_dict in line["passage"]["questions"]:
+                question = question_dict["question"]
+                question_id = question_dict["idx"]
+                answer_dicts = question_dict["answers"]
+                examples.append(
+                    Example(
+                        guid="%s-%s-%s" % (set_type, passage_id, question_id),
+                        prompt=passage,
+                        choice_list=[
+                            question + " " + answer_dict["text"] for answer_dict in answer_dicts
+                        ],
+                        label=answer_dicts[1]["label"] == "True"
+                        if set_type != "test"
+                        else cls.CHOICE_KEYS[-1],
+                    )
+                )
+
+        return examples

jiant/tasks/retrieval.py

@@ -62,6 +62,7 @@
 from jiant.tasks.lib.wsc import WSCTask
 from jiant.tasks.lib.xnli import XnliTask
 from jiant.tasks.lib.xquad import XquadTask
+from jiant.tasks.lib.mcscript import MCScriptTask
 from jiant.tasks.lib.arct import ArctTask
 
 from jiant.tasks.core import Task
@@ -130,6 +131,7 @@
     "wsc": WSCTask,
     "xnli": XnliTask,
     "xquad": XquadTask,
+    "mcscript": MCScriptTask,
     "arct": ArctTask,
 }