diff --git a/lmms_eval/tasks/mmupd/_default_template_mmaad_instruction_yaml b/lmms_eval/tasks/mmupd/_default_template_mmaad_instruction_yaml deleted file mode 100644 index fe4c8148..00000000 --- a/lmms_eval/tasks/mmupd/_default_template_mmaad_instruction_yaml +++ /dev/null @@ -1,23 +0,0 @@ -dataset_path: MM-UPD/MM-UPD -doc_to_target: "answer" -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nIf all the options are incorrect, answer \"F. None of the above\"." -doc_to_visual: !function utils.mmupd_doc_to_visual -doc_to_text: !function utils.mmupd_doc_to_text -doc_to_target: "answer" -process_results: !function utils.mmupd_process_results -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original -output_type: generate_until -dataset_name: mmaad_base -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false diff --git a/lmms_eval/tasks/mmupd/_default_template_mmaad_option_yaml b/lmms_eval/tasks/mmupd/_default_template_mmaad_option_yaml deleted file mode 100644 index aa8b9c35..00000000 --- a/lmms_eval/tasks/mmupd/_default_template_mmaad_option_yaml +++ /dev/null @@ -1,23 +0,0 @@ -dataset_path: MM-UPD/MM-UPD -doc_to_target: "answer" -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer with the option's letter from the given choices directly." -doc_to_visual: !function utils.mmupd_doc_to_visual -doc_to_text: !function utils.mmupd_doc_to_text -doc_to_target: "answer" -process_results: !function utils.mmupd_process_results -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original -output_type: generate_until -dataset_name: mmaad_option -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false diff --git a/lmms_eval/tasks/mmupd/_default_template_mmiasd_base_yaml b/lmms_eval/tasks/mmupd/_default_template_mmiasd_base_yaml deleted file mode 100644 index 124c9007..00000000 --- a/lmms_eval/tasks/mmupd/_default_template_mmiasd_base_yaml +++ /dev/null @@ -1,23 +0,0 @@ -dataset_path: MM-UPD/MM-UPD -doc_to_target: "answer" -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\n" -doc_to_visual: !function utils.mmupd_doc_to_visual -doc_to_text: !function utils.mmupd_doc_to_text -doc_to_target: "answer" -process_results: !function utils.mmupd_process_results -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original -output_type: generate_until -dataset_name: mmiasd_base -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false diff --git a/lmms_eval/tasks/mmupd/_default_template_mmiasd_instruction_yaml b/lmms_eval/tasks/mmupd/_default_template_mmiasd_instruction_yaml deleted file mode 100644 index b8d1e473..00000000 --- a/lmms_eval/tasks/mmupd/_default_template_mmiasd_instruction_yaml +++ /dev/null @@ -1,23 +0,0 @@ -dataset_path: MM-UPD/MM-UPD -doc_to_target: "answer" -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nIf all the options are incorrect, answer \"F. None of the above\"." -doc_to_visual: !function utils.mmupd_doc_to_visual -doc_to_text: !function utils.mmupd_doc_to_text -doc_to_target: "answer" -process_results: !function utils.mmupd_process_results -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original -output_type: generate_until -dataset_name: mmiasd_base -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false diff --git a/lmms_eval/tasks/mmupd/_default_template_mmiasd_option_yaml b/lmms_eval/tasks/mmupd/_default_template_mmiasd_option_yaml deleted file mode 100644 index 3e40e8da..00000000 --- a/lmms_eval/tasks/mmupd/_default_template_mmiasd_option_yaml +++ /dev/null @@ -1,23 +0,0 @@ -dataset_path: MM-UPD/MM-UPD -doc_to_target: "answer" -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer with the option's letter from the given choices directly." -doc_to_visual: !function utils.mmupd_doc_to_visual -doc_to_text: !function utils.mmupd_doc_to_text -doc_to_target: "answer" -process_results: !function utils.mmupd_process_results -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original -output_type: generate_until -dataset_name: mmiasd_option -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false diff --git a/lmms_eval/tasks/mmupd/_default_template_mmivqd_base_yaml b/lmms_eval/tasks/mmupd/_default_template_mmivqd_base_yaml deleted file mode 100644 index 986ae379..00000000 --- a/lmms_eval/tasks/mmupd/_default_template_mmivqd_base_yaml +++ /dev/null @@ -1,23 +0,0 @@ -dataset_path: MM-UPD/MM-UPD -doc_to_target: "answer" -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\n" -doc_to_visual: !function utils.mmupd_doc_to_visual -doc_to_text: !function utils.mmupd_doc_to_text -doc_to_target: "answer" -process_results: !function utils.mmupd_process_results -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original -output_type: generate_until -dataset_name: mmivqd_base -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false diff --git a/lmms_eval/tasks/mmupd/_default_template_mmivqd_instruction_yaml b/lmms_eval/tasks/mmupd/_default_template_mmivqd_instruction_yaml deleted file mode 100644 index 8362657e..00000000 --- a/lmms_eval/tasks/mmupd/_default_template_mmivqd_instruction_yaml +++ /dev/null @@ -1,23 +0,0 @@ -dataset_path: MM-UPD/MM-UPD -doc_to_target: "answer" -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nIf the given image is irrelevant to the question, answer \"F. The image and question are irrelevant.\"." -doc_to_visual: !function utils.mmupd_doc_to_visual -doc_to_text: !function utils.mmupd_doc_to_text -doc_to_target: "answer" -process_results: !function utils.mmupd_process_results -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original -output_type: generate_until -dataset_name: mmivqd_base -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false diff --git a/lmms_eval/tasks/mmupd/_default_template_mmivqd_option_yaml b/lmms_eval/tasks/mmupd/_default_template_mmivqd_option_yaml deleted file mode 100644 index 14778cac..00000000 --- a/lmms_eval/tasks/mmupd/_default_template_mmivqd_option_yaml +++ /dev/null @@ -1,23 +0,0 @@ -dataset_path: MM-UPD/MM-UPD -doc_to_target: "answer" -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\nAnswer with the option's letter from the given choices directly." -doc_to_visual: !function utils.mmupd_doc_to_visual -doc_to_text: !function utils.mmupd_doc_to_text -doc_to_target: "answer" -process_results: !function utils.mmupd_process_results -model_specific_generation_kwargs: - llava: - image_aspect_ratio: original -output_type: generate_until -dataset_name: mmivqd_option -generation_kwargs: - until: - - "ASSISTANT:" - max_new_tokens: 1024 - temperature: 0 - top_p: 0 - num_beams: 1 - do_sample: false diff --git a/lmms_eval/tasks/mmupd/_default_template_mmaad_base_yaml b/lmms_eval/tasks/mmupd/_default_template_mmupd_yaml similarity index 81% rename from lmms_eval/tasks/mmupd/_default_template_mmaad_base_yaml rename to lmms_eval/tasks/mmupd/_default_template_mmupd_yaml index bbe9a9cf..7aa8d812 100644 --- a/lmms_eval/tasks/mmupd/_default_template_mmaad_base_yaml +++ b/lmms_eval/tasks/mmupd/_default_template_mmupd_yaml @@ -1,9 +1,5 @@ dataset_path: MM-UPD/MM-UPD doc_to_target: "answer" -model_specific_prompt_kwargs: - default: - pre_prompt: "" - post_prompt: "\n" doc_to_visual: !function utils.mmupd_doc_to_visual doc_to_text: !function utils.mmupd_doc_to_text doc_to_target: "answer" @@ -12,7 +8,6 @@ model_specific_generation_kwargs: llava: image_aspect_ratio: original output_type: generate_until -dataset_name: mmaad_base generation_kwargs: until: - "ASSISTANT:" diff --git a/lmms_eval/tasks/mmupd/mmaad_base.yaml b/lmms_eval/tasks/mmupd/mmaad_base.yaml index dd38d33e..9a66b3e8 100644 --- a/lmms_eval/tasks/mmupd/mmaad_base.yaml +++ b/lmms_eval/tasks/mmupd/mmaad_base.yaml @@ -1,6 +1,11 @@ task: "mmaad_base" test_split: test -include: _default_template_mmaad_base_yaml +dataset_name: mmaad_base +model_specific_prompt_kwargs: + default: + pre_prompt: "" + post_prompt: "\n" +include: _default_template_mmupd_yaml metric_list: - metric: gpt_eval_score aggregation: !function utils.mmaad_base diff --git a/lmms_eval/tasks/mmupd/mmaad_instruction.yaml b/lmms_eval/tasks/mmupd/mmaad_instruction.yaml index 384425d3..30a3bac1 100644 --- a/lmms_eval/tasks/mmupd/mmaad_instruction.yaml +++ b/lmms_eval/tasks/mmupd/mmaad_instruction.yaml @@ -1,6 +1,11 @@ task: "mmaad_instruction" test_split: test -include: _default_template_mmaad_instruction_yaml +dataset_name: mmaad_base +model_specific_prompt_kwargs: + default: + pre_prompt: "" + post_prompt: "\nIf all the options are incorrect, answer \"F. None of the above\"." +include: _default_template_mmupd_yaml metric_list: - metric: gpt_eval_score aggregation: !function utils.mmaad_instruction diff --git a/lmms_eval/tasks/mmupd/mmaad_option.yaml b/lmms_eval/tasks/mmupd/mmaad_option.yaml index 81ac6a02..f110b822 100644 --- a/lmms_eval/tasks/mmupd/mmaad_option.yaml +++ b/lmms_eval/tasks/mmupd/mmaad_option.yaml @@ -1,6 +1,11 @@ task: "mmaad_option" test_split: test -include: _default_template_mmaad_option_yaml +dataset_name: mmaad_option +model_specific_prompt_kwargs: + default: + pre_prompt: "" + post_prompt: "\nAnswer with the option's letter from the given choices directly." +include: _default_template_mmupd_yaml metric_list: - metric: gpt_eval_score aggregation: !function utils.mmaad_option diff --git a/lmms_eval/tasks/mmupd/mmiasd_base.yaml b/lmms_eval/tasks/mmupd/mmiasd_base.yaml index 29468358..6a9159fd 100644 --- a/lmms_eval/tasks/mmupd/mmiasd_base.yaml +++ b/lmms_eval/tasks/mmupd/mmiasd_base.yaml @@ -1,6 +1,11 @@ task: "mmiasd_base" test_split: test -include: _default_template_mmiasd_base_yaml +dataset_name: mmiasd_base +model_specific_prompt_kwargs: + default: + pre_prompt: "" + post_prompt: "\n" +include: _default_template_mmupd_yaml metric_list: - metric: gpt_eval_score aggregation: !function utils.mmiasd_base diff --git a/lmms_eval/tasks/mmupd/mmiasd_instruction.yaml b/lmms_eval/tasks/mmupd/mmiasd_instruction.yaml index a6406a1a..d0ac3a00 100644 --- a/lmms_eval/tasks/mmupd/mmiasd_instruction.yaml +++ b/lmms_eval/tasks/mmupd/mmiasd_instruction.yaml @@ -1,6 +1,11 @@ task: "mmiasd_instruction" test_split: test -include: _default_template_mmiasd_instruction_yaml +dataset_name: mmiasd_base +model_specific_prompt_kwargs: + default: + pre_prompt: "" + post_prompt: "\nIf all the options are incorrect, answer \"F. None of the above\"." +include: _default_template_mmupd_yaml metric_list: - metric: gpt_eval_score aggregation: !function utils.mmiasd_instruction diff --git a/lmms_eval/tasks/mmupd/mmiasd_option.yaml b/lmms_eval/tasks/mmupd/mmiasd_option.yaml index 83e3248f..a1a1aead 100644 --- a/lmms_eval/tasks/mmupd/mmiasd_option.yaml +++ b/lmms_eval/tasks/mmupd/mmiasd_option.yaml @@ -1,6 +1,11 @@ task: "mmiasd_option" test_split: test -include: _default_template_mmiasd_option_yaml +dataset_name: mmiasd_option +model_specific_prompt_kwargs: + default: + pre_prompt: "" + post_prompt: "\nAnswer with the option's letter from the given choices directly." +include: _default_template_mmupd_yaml metric_list: - metric: gpt_eval_score aggregation: !function utils.mmiasd_option diff --git a/lmms_eval/tasks/mmupd/mmivqd_base.yaml b/lmms_eval/tasks/mmupd/mmivqd_base.yaml index 80b856f5..82853fde 100644 --- a/lmms_eval/tasks/mmupd/mmivqd_base.yaml +++ b/lmms_eval/tasks/mmupd/mmivqd_base.yaml @@ -1,6 +1,11 @@ task: "mmivqd_base" test_split: test -include: _default_template_mmivqd_base_yaml +dataset_name: mmivqd_base +model_specific_prompt_kwargs: + default: + pre_prompt: "" + post_prompt: "\n" +include: _default_template_mmupd_yaml metric_list: - metric: gpt_eval_score aggregation: !function utils.mmivqd_base diff --git a/lmms_eval/tasks/mmupd/mmivqd_instruction.yaml b/lmms_eval/tasks/mmupd/mmivqd_instruction.yaml index ef0d3604..a024022a 100644 --- a/lmms_eval/tasks/mmupd/mmivqd_instruction.yaml +++ b/lmms_eval/tasks/mmupd/mmivqd_instruction.yaml @@ -1,6 +1,11 @@ task: "mmivqd_instruction" test_split: test -include: _default_template_mmivqd_instruction_yaml +dataset_name: mmivqd_base +model_specific_prompt_kwargs: + default: + pre_prompt: "" + post_prompt: "\nIf the given image is irrelevant to the question, answer \"F. The image and question are irrelevant.\"." +include: _default_template_mmupd_yaml metric_list: - metric: gpt_eval_score aggregation: !function utils.mmivqd_instruction diff --git a/lmms_eval/tasks/mmupd/mmivqd_option.yaml b/lmms_eval/tasks/mmupd/mmivqd_option.yaml index 6b1d2cc9..1363c345 100644 --- a/lmms_eval/tasks/mmupd/mmivqd_option.yaml +++ b/lmms_eval/tasks/mmupd/mmivqd_option.yaml @@ -1,6 +1,11 @@ task: "mmivqd_option" test_split: test -include: _default_template_mmivqd_option_yaml +dataset_name: mmivqd_option +model_specific_prompt_kwargs: + default: + pre_prompt: "" + post_prompt: "\nAnswer with the option's letter from the given choices directly." +include: _default_template_mmupd_yaml metric_list: - metric: gpt_eval_score aggregation: !function utils.mmivqd_option