diff --git a/promptsource/templates.py b/promptsource/templates.py index 2b9eb341e..2c9785c40 100644 --- a/promptsource/templates.py +++ b/promptsource/templates.py @@ -27,7 +27,20 @@ # These are users whose datasets should be included in the results returned by # filter_english_datasets (regardless of their metadata) -INCLUDED_USERS = {"Zaid", "craffel", "GEM", "aps", "khalidalt", "shanya", "rbawden", "BigScienceBiasEval", "gsarti"} +INCLUDED_USERS = { + "Zaid", + "craffel", + "GEM", + "aps", + "khalidalt", + "shanya", + "rbawden", + "BigScienceBiasEval", + "gsarti", + "Helsinki-NLP", + "Muennighoff", + "facebook", +} # These are the metrics with which templates can be tagged METRICS = { diff --git a/promptsource/templates/Muennighoff/xstory_cloze/en/templates.yaml b/promptsource/templates/Muennighoff/xstory_cloze/en/templates.yaml new file mode 100644 index 000000000..1b30ef865 --- /dev/null +++ b/promptsource/templates/Muennighoff/xstory_cloze/en/templates.yaml @@ -0,0 +1,99 @@ +dataset: Muennighoff/xstory_cloze +subset: en +templates: + 2a4946f9-a0e2-4fbb-aee8-b26ead2cf6b8: !Template + answer_choices: '{{sentence_quiz1}} ||| {{sentence_quiz2}}' + id: 2a4946f9-a0e2-4fbb-aee8-b26ead2cf6b8 + jinja: '{{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} + What is a possible continuation for the story given the following options ? + - {{answer_choices | join("\n- ")}} ||| {{answer_choices[answer_right_ending + -1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Answer Given options + reference: '' + 2a9d53bc-eb77-4e7c-af6e-3d15b79d6cf1: !Template + answer_choices: '{{sentence_quiz1}} ||| {{sentence_quiz2}}' + id: 2a9d53bc-eb77-4e7c-af6e-3d15b79d6cf1 + jinja: "Read the following story :\n\n{{input_sentence_1}}\n{{input_sentence_2}}\n\ + {{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose a possible ending for the\ + \ previous story from the following options: \n- {{answer_choices | join(\"\\\ + n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Choose Story Ending + reference: '' + 8dab69d1-cad0-4d2f-a7cc-120df233571c: !Template + answer_choices: '{{sentence_quiz1}} ||| {{sentence_quiz2}}' + id: 8dab69d1-cad0-4d2f-a7cc-120df233571c + jinja: 'Yesterday, I watched a movie. Here''s what happened: {{input_sentence_1}} + {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What happens + next? - {{answer_choices | join("\n- ")}} ||| {{answer_choices[answer_right_ending + -1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Movie What Happens Next + reference: '' + c5c8445f-2d3a-4691-bdd5-58956816702f: !Template + answer_choices: '{{sentence_quiz1}} ||| {{sentence_quiz2}}' + id: c5c8445f-2d3a-4691-bdd5-58956816702f + jinja: "What is a possible continuation for the following story ? \n\n{{input_sentence_1}}\n\ + {{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose from\ + \ the following options:\n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending\ + \ -1]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Story Continuation and Options + reference: '' + caffa716-43cf-4954-a35c-655d775321e6: !Template + answer_choices: '{{sentence_quiz1}} ||| {{sentence_quiz2}}' + id: caffa716-43cf-4954-a35c-655d775321e6 + jinja: 'Generate a possible ending for the following story: {{input_sentence_1}} + {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} ||| {{answer_choices[answer_right_ending + -1]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - BLEU + - ROUGE + original_task: false + name: Generate Ending + reference: '' + d6f3d802-4f97-449f-a911-03470d418f7d: !Template + answer_choices: '{{sentence_quiz1}} ||| {{sentence_quiz2}}' + id: d6f3d802-4f97-449f-a911-03470d418f7d + jinja: 'I read the following novel: {{input_sentence_1}} {{input_sentence_2}} + {{input_sentence_3}} {{input_sentence_4}} What do you think is the most probable + ending? You can choose from the following options: - {{answer_choices | join("\n- + ")}} ||| {{answer_choices[answer_right_ending -1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Novel Correct Ending + reference: ''