merge model_specific_prompt_kwargs and dataset_name into each task yaml

EvolvingLMMs-Lab · May 31, 2024 · 0581ab3 · 0581ab3
1 parent d224794
commit 0581ab3
Show file tree

Hide file tree

Showing 18 changed files with 54 additions and 198 deletions.
diff --git a/lmms_eval/tasks/mmupd/_default_template_mmaad_instruction_yaml b/lmms_eval/tasks/mmupd/_default_template_mmaad_instruction_yaml
diff --git a/lmms_eval/tasks/mmupd/_default_template_mmaad_option_yaml b/lmms_eval/tasks/mmupd/_default_template_mmaad_option_yaml
diff --git a/lmms_eval/tasks/mmupd/_default_template_mmiasd_base_yaml b/lmms_eval/tasks/mmupd/_default_template_mmiasd_base_yaml
diff --git a/lmms_eval/tasks/mmupd/_default_template_mmiasd_instruction_yaml b/lmms_eval/tasks/mmupd/_default_template_mmiasd_instruction_yaml
diff --git a/lmms_eval/tasks/mmupd/_default_template_mmiasd_option_yaml b/lmms_eval/tasks/mmupd/_default_template_mmiasd_option_yaml
diff --git a/lmms_eval/tasks/mmupd/_default_template_mmivqd_base_yaml b/lmms_eval/tasks/mmupd/_default_template_mmivqd_base_yaml
diff --git a/lmms_eval/tasks/mmupd/_default_template_mmivqd_instruction_yaml b/lmms_eval/tasks/mmupd/_default_template_mmivqd_instruction_yaml
diff --git a/lmms_eval/tasks/mmupd/_default_template_mmivqd_option_yaml b/lmms_eval/tasks/mmupd/_default_template_mmivqd_option_yaml
diff --git a/...s/mmupd/_default_template_mmaad_base_yaml → .../tasks/mmupd/_default_template_mmupd_yaml b/...s/mmupd/_default_template_mmaad_base_yaml → .../tasks/mmupd/_default_template_mmupd_yaml
@@ -1,9 +1,5 @@
 dataset_path: MM-UPD/MM-UPD
 doc_to_target: "answer"
-model_specific_prompt_kwargs:
-  default:
-    pre_prompt: ""
-    post_prompt: "\n"
 doc_to_visual: !function utils.mmupd_doc_to_visual
 doc_to_text: !function utils.mmupd_doc_to_text
 doc_to_target: "answer"
@@ -12,7 +8,6 @@ model_specific_generation_kwargs:
   llava:
     image_aspect_ratio: original
 output_type: generate_until
-dataset_name: mmaad_base
 generation_kwargs:
   until:
     - "ASSISTANT:"

diff --git a/lmms_eval/tasks/mmupd/mmaad_base.yaml b/lmms_eval/tasks/mmupd/mmaad_base.yaml
@@ -1,6 +1,11 @@
 task: "mmaad_base"
 test_split: test
-include: _default_template_mmaad_base_yaml
+dataset_name: mmaad_base
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\n"
+include: _default_template_mmupd_yaml
 metric_list:
   - metric: gpt_eval_score
     aggregation: !function utils.mmaad_base

diff --git a/lmms_eval/tasks/mmupd/mmaad_instruction.yaml b/lmms_eval/tasks/mmupd/mmaad_instruction.yaml
@@ -1,6 +1,11 @@
 task: "mmaad_instruction"
 test_split: test
-include: _default_template_mmaad_instruction_yaml
+dataset_name: mmaad_base
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nIf all the options are incorrect, answer \"F. None of the above\"."
+include: _default_template_mmupd_yaml
 metric_list:
   - metric: gpt_eval_score
     aggregation: !function utils.mmaad_instruction

diff --git a/lmms_eval/tasks/mmupd/mmaad_option.yaml b/lmms_eval/tasks/mmupd/mmaad_option.yaml
@@ -1,6 +1,11 @@
 task: "mmaad_option"
 test_split: test
-include: _default_template_mmaad_option_yaml
+dataset_name: mmaad_option
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer with the option's letter from the given choices directly."
+include: _default_template_mmupd_yaml
 metric_list:
   - metric: gpt_eval_score
     aggregation: !function utils.mmaad_option

diff --git a/lmms_eval/tasks/mmupd/mmiasd_base.yaml b/lmms_eval/tasks/mmupd/mmiasd_base.yaml
@@ -1,6 +1,11 @@
 task: "mmiasd_base"
 test_split: test
-include: _default_template_mmiasd_base_yaml
+dataset_name: mmiasd_base
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\n"
+include: _default_template_mmupd_yaml
 metric_list:
   - metric: gpt_eval_score
     aggregation: !function utils.mmiasd_base

diff --git a/lmms_eval/tasks/mmupd/mmiasd_instruction.yaml b/lmms_eval/tasks/mmupd/mmiasd_instruction.yaml
@@ -1,6 +1,11 @@
 task: "mmiasd_instruction"
 test_split: test
-include: _default_template_mmiasd_instruction_yaml
+dataset_name: mmiasd_base
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nIf all the options are incorrect, answer \"F. None of the above\"."
+include: _default_template_mmupd_yaml
 metric_list:
   - metric: gpt_eval_score
     aggregation: !function utils.mmiasd_instruction

diff --git a/lmms_eval/tasks/mmupd/mmiasd_option.yaml b/lmms_eval/tasks/mmupd/mmiasd_option.yaml
@@ -1,6 +1,11 @@
 task: "mmiasd_option"
 test_split: test
-include: _default_template_mmiasd_option_yaml
+dataset_name: mmiasd_option
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer with the option's letter from the given choices directly."
+include: _default_template_mmupd_yaml
 metric_list:
   - metric: gpt_eval_score
     aggregation: !function utils.mmiasd_option

diff --git a/lmms_eval/tasks/mmupd/mmivqd_base.yaml b/lmms_eval/tasks/mmupd/mmivqd_base.yaml
@@ -1,6 +1,11 @@
 task: "mmivqd_base"
 test_split: test
-include: _default_template_mmivqd_base_yaml
+dataset_name: mmivqd_base
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\n"
+include: _default_template_mmupd_yaml
 metric_list:
   - metric: gpt_eval_score
     aggregation: !function utils.mmivqd_base

diff --git a/lmms_eval/tasks/mmupd/mmivqd_instruction.yaml b/lmms_eval/tasks/mmupd/mmivqd_instruction.yaml
@@ -1,6 +1,11 @@
 task: "mmivqd_instruction"
 test_split: test
-include: _default_template_mmivqd_instruction_yaml
+dataset_name: mmivqd_base
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nIf the given image is irrelevant to the question, answer \"F. The image and question are irrelevant.\"."
+include: _default_template_mmupd_yaml
 metric_list:
   - metric: gpt_eval_score
     aggregation: !function utils.mmivqd_instruction

diff --git a/lmms_eval/tasks/mmupd/mmivqd_option.yaml b/lmms_eval/tasks/mmupd/mmivqd_option.yaml
@@ -1,6 +1,11 @@
 task: "mmivqd_option"
 test_split: test
-include: _default_template_mmivqd_option_yaml
+dataset_name: mmivqd_option
+model_specific_prompt_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer with the option's letter from the given choices directly."
+include: _default_template_mmupd_yaml
 metric_list:
   - metric: gpt_eval_score
     aggregation: !function utils.mmivqd_option