Fixing senteval-probing preprocessing

jiant/tasks/senteval_probing.py

@@ -272,28 +272,40 @@ def __init__(self, path, max_seq_len, name, **kw):
         self.train_data_text = None
         self.val_data_text = None
         self.test_data_text = None
+        self.labels = []
 
     def get_all_labels(self):
         return list(set(self.labels))
 
     def get_sentences(self):
         return self.sentences
 
+    def process_split(self, split, indexers, model_preprocessing_interface):
+        return process_single_pair_task_split(
+            split,
+            indexers,
+            model_preprocessing_interface,
+            label_namespace=self._label_namespace,
+            is_pair=False,
+            skip_indexing=False,
+        )
+
     def load_data(self):
         """ Load data """
 
         def load_csv(data_file):
             rows = pd.read_csv(data_file, encoding="utf-8")
-            rows["s1"] = rows["2"].apply(
+            labels = rows["1"].apply(lambda x: x.split("\t")[0])
+            s1 = rows["1"].apply(lambda x: x.split("\t")[1])
+            s1 = s1.apply(
                 lambda x: tokenize_and_truncate(self._tokenizer_name, x, self.max_seq_len)
             )
-            self.labels.append(rows["1"].tolist())
-            return rows["s1"].tolist(), [], rows["1"].tolist(), list(range(len(rows)))
+            self.labels = list(set(labels.tolist()))
+            return s1.tolist(), [], labels.tolist(), list(range(len(rows)))
 
         self.train_data_text = load_csv(os.path.join(self.path, "train.csv"))
         self.val_data_text = load_csv(os.path.join(self.path, "val.csv"))
         self.test_data_text = load_csv(os.path.join(self.path, "test.csv"))
-
         sentences = []
         for split in ["train", "val", "test"]:
             split_data = getattr(self, "%s_data_text" % split)

jiant/trainer.py

@@ -951,7 +951,11 @@ def _validate(self, val_pass, tasks, batch_size, periodic_save=True):
 
         # Get validation numbers for each task
         for task in tasks:
-            n_examples_overall, task_infos, all_val_metrics = self._calculate_validation_performance(  # noqa
+            (
+                n_examples_overall,
+                task_infos,
+                all_val_metrics,
+            ) = self._calculate_validation_performance(
                 task, task_infos, tasks, batch_size, all_val_metrics, n_examples_overall
             )
         # scale the micro avg contributions w/ total size of validation set.

scripts/preprocess_senteval_probing.py

@@ -9,21 +9,21 @@
 
 
 def parse_senteval_probing(args):
-    files = [x for x in os.listdir(args.senteval_path) if "txt" in x]
+    files = [x for x in os.listdir(args.senteval_probing_path) if "txt" in x]
     for file in files:
-        file_pd = pd.read_fwf(os.path.join(args.senteval_path, file), header=None)
+        file_pd = pd.read_fwf(os.path.join(args.senteval_probing_path, file), header=None)
         files_train = file_pd[file_pd[0] == "tr"]
         task_name = file.split(".")[0]
-        if not os.path.exists(os.path.join(args.senteval_path, task_name)):
-            os.mkdir(os.path.join(args.senteval_path, task_name))
-        files_train.to_csv(os.path.join(args.senteval_path, task_name, "train.tsv"))
+        if not os.path.exists(os.path.join(args.senteval_probing_path, task_name)):
+            os.mkdir(os.path.join(args.senteval_probing_path, task_name))
+        files_train.to_csv(os.path.join(args.senteval_probing_path, task_name, "train.csv"))
         files_val = file_pd[file_pd[0] == "va"]
         task_name = file.split(".")[0]
-        files_val.to_csv(os.path.join(args.senteval_path, task_name, "val.tsv"))
+        files_val.to_csv(os.path.join(args.senteval_probing_path, task_name, "val.csv"))
 
         files_test = file_pd[file_pd[0] == "te"]
         task_name = file.split(".")[0]
-        files_test.to_csv(os.path.join(args.senteval_path, task_name, "test.tsv"))
+        files_test.to_csv(os.path.join(args.senteval_probing_path, task_name, "test.csv"))
 
 
 parser = argparse.ArgumentParser()