Fix prompts table benchmark (#1581)

* add metrics to scigen

Signed-off-by: ShirApp <shirashury@gmail.com>

* improve the instructions in table benchmark datasets + add postprocessing for using first line

Signed-off-by: ShirApp <shirashury@gmail.com>

* rerun prepare

Signed-off-by: ShirApp <shirashury@gmail.com>

* add template index 0 to all benchmark recipes (so the prompt in the card will be used)

Signed-off-by: ShirApp <shirashury@gmail.com>

---------

Signed-off-by: ShirApp <shirashury@gmail.com>

prepare/cards/wikitq.py

@@ -25,8 +25,11 @@
     task="tasks.qa.extractive[metrics=[metrics.f1_strings, metrics.unsorted_list_exact_match]]",
     templates=[
         MultiReferenceTemplate(
-            instruction="Answer the question based on the provided table. Extract and output only the final answer—the exact phrase or data from the table that directly answers the question. Do not include any alterations, explanations, or introductory text"
-            + "\nHere are some input-output examples. Read the examples carefully to figure out the mapping. The output of the last example is not given, and your job is to figure out what it is.",
+            instruction="Answer the question based on the provided table. "
+            "Extract and output only the final answer—the exact phrase or data from the table that directly answers the question. "
+            "Do not include any alterations, explanations, or introductory text."
+            "\nHere are some input-output examples. Read the examples carefully to figure out the mapping. "
+            "The output of the last example is not given, and your job is to figure out what it is.",
             input_format="\nQuestion: {question}" "\nTable: {context}" "\nAnswer: ",
             references_field="answers",
             postprocessors=[

prepare/recipes/tables_benchmark.py

@@ -43,6 +43,7 @@
                 "num_demos": num_demos,
                 "demos_pool_size": DEMOS_POOL_SIZE,
                 "loader_limit": LOADER_LIMIT,
+                "template_card_index": 0,
                 "augmentor": [f"augmentors.table.{a!s}" for a in augment]
                 if augment
                 else None,

src/unitxt/catalog/cards/wikitq.json

@@ -37,7 +37,7 @@
     "templates": [
         {
             "__type__": "multi_reference_template",
-            "instruction": "Answer the question based on the provided table. Extract and output only the final answer—the exact phrase or data from the table that directly answers the question. Do not include any alterations, explanations, or introductory text\nHere are some input-output examples. Read the examples carefully to figure out the mapping. The output of the last example is not given, and your job is to figure out what it is.",
+            "instruction": "Answer the question based on the provided table. Extract and output only the final answer—the exact phrase or data from the table that directly answers the question. Do not include any alterations, explanations, or introductory text.\nHere are some input-output examples. Read the examples carefully to figure out the mapping. The output of the last example is not given, and your job is to figure out what it is.",
             "input_format": "\nQuestion: {question}\nTable: {context}\nAnswer: ",
             "references_field": "answers",
             "postprocessors": [

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/concat/insert_empty_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.insert_empty_rows[times=2]"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/concat/no_augmentation_5_demos.json

@@ -5,5 +5,6 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": null
 }

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/concat/shuffle_cols_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_cols"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/concat/shuffle_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_rows"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/concat/transpose_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.transpose"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/csv/insert_empty_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.insert_empty_rows[times=2]"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/csv/no_augmentation_5_demos.json

@@ -5,5 +5,6 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": null
 }

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/csv/shuffle_cols_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_cols"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/csv/shuffle_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_rows"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/csv/transpose_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.transpose"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/df/insert_empty_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.insert_empty_rows[times=2]"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/df/no_augmentation_5_demos.json

@@ -5,5 +5,6 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": null
 }

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/df/shuffle_cols_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_cols"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/df/shuffle_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_rows"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/df/transpose_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.transpose"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/html/insert_empty_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.insert_empty_rows[times=2]"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/html/no_augmentation_5_demos.json

@@ -5,5 +5,6 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": null
 }

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/html/shuffle_cols_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_cols"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/html/shuffle_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_rows"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/html/transpose_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.transpose"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/indexed_row_major/insert_empty_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.insert_empty_rows[times=2]"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/indexed_row_major/no_augmentation_5_demos.json

@@ -5,5 +5,6 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": null
 }

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/indexed_row_major/shuffle_cols_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_cols"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/indexed_row_major/shuffle_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_rows"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/indexed_row_major/transpose_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.transpose"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/json/insert_empty_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.insert_empty_rows[times=2]"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/json/no_augmentation_5_demos.json

@@ -5,5 +5,6 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": null
 }

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/json/shuffle_cols_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_cols"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/json/shuffle_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_rows"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/json/transpose_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.transpose"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/markdown/insert_empty_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.insert_empty_rows[times=2]"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/markdown/no_augmentation_5_demos.json

@@ -5,5 +5,6 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": null
 }

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/markdown/shuffle_cols_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_cols"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/markdown/shuffle_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_rows"
     ]

src/unitxt/catalog/recipes/tables_benchmark/fin_qa/markdown/transpose_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.transpose"
     ]

src/unitxt/catalog/recipes/tables_benchmark/numeric_nlg/concat/insert_empty_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.insert_empty_rows[times=2]"
     ]

src/unitxt/catalog/recipes/tables_benchmark/numeric_nlg/concat/no_augmentation_5_demos.json

@@ -5,5 +5,6 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": null
 }

src/unitxt/catalog/recipes/tables_benchmark/numeric_nlg/concat/shuffle_cols_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_cols"
     ]

src/unitxt/catalog/recipes/tables_benchmark/numeric_nlg/concat/shuffle_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_rows"
     ]

src/unitxt/catalog/recipes/tables_benchmark/numeric_nlg/concat/transpose_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.transpose"
     ]

src/unitxt/catalog/recipes/tables_benchmark/numeric_nlg/csv/insert_empty_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.insert_empty_rows[times=2]"
     ]

src/unitxt/catalog/recipes/tables_benchmark/numeric_nlg/csv/no_augmentation_5_demos.json

@@ -5,5 +5,6 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": null
 }

src/unitxt/catalog/recipes/tables_benchmark/numeric_nlg/csv/shuffle_cols_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_cols"
     ]

src/unitxt/catalog/recipes/tables_benchmark/numeric_nlg/csv/shuffle_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.shuffle_rows"
     ]

src/unitxt/catalog/recipes/tables_benchmark/numeric_nlg/csv/transpose_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.transpose"
     ]

src/unitxt/catalog/recipes/tables_benchmark/numeric_nlg/df/insert_empty_rows_augmentation_5_demos.json

@@ -5,6 +5,7 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": [
         "augmentors.table.insert_empty_rows[times=2]"
     ]

src/unitxt/catalog/recipes/tables_benchmark/numeric_nlg/df/no_augmentation_5_demos.json

@@ -5,5 +5,6 @@
     "num_demos": 5,
     "demos_pool_size": -1,
     "loader_limit": 10000,
+    "template_card_index": 0,
     "augmentor": null
 }