From d74336f3fca6ee031e369301145b4bec5856c648 Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Wed, 17 Aug 2022 17:57:22 +0200 Subject: [PATCH 1/6] Add MultiEURLEX --- .../multi_eurlex/all_languages/templates.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 promptsource/templates/multi_eurlex/all_languages/templates.yaml diff --git a/promptsource/templates/multi_eurlex/all_languages/templates.yaml b/promptsource/templates/multi_eurlex/all_languages/templates.yaml new file mode 100644 index 000000000..f09a0f334 --- /dev/null +++ b/promptsource/templates/multi_eurlex/all_languages/templates.yaml @@ -0,0 +1,18 @@ +dataset: multi_eurlex +subset: all_languages +templates: + 28ddea55-1710-4615-bbfa-fe5803e21e43: !Template + answer_choices: null + id: 28ddea55-1710-4615-bbfa-fe5803e21e43 + jinja: 'If the French version says: {{text["fr"]}}; then the English version + should say: + + ||| {{text["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: version-fr-en-source+target + reference: '' From 260ff2969fd7718bc1f228947c7f0ea564bb51e8 Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Wed, 17 Aug 2022 19:25:02 +0200 Subject: [PATCH 2/6] Add prompts --- .../multi_eurlex/all_languages/templates.yaml | 69 ++++++++++++++++++- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/promptsource/templates/multi_eurlex/all_languages/templates.yaml b/promptsource/templates/multi_eurlex/all_languages/templates.yaml index f09a0f334..11dfaa26a 100644 --- a/promptsource/templates/multi_eurlex/all_languages/templates.yaml +++ b/promptsource/templates/multi_eurlex/all_languages/templates.yaml @@ -1,12 +1,11 @@ dataset: multi_eurlex subset: all_languages templates: - 28ddea55-1710-4615-bbfa-fe5803e21e43: !Template + 38ddea55-1710-4615-bbfa-fe5803e21e43: !Template answer_choices: null - id: 28ddea55-1710-4615-bbfa-fe5803e21e43 + id: 38ddea55-1710-4615-bbfa-fe5803e21e43 jinja: 'If the French version says: {{text["fr"]}}; then the English version should say: - ||| {{text["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false @@ -16,3 +15,67 @@ templates: original_task: true name: version-fr-en-source+target reference: '' + 2bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02: !Template + answer_choices: null + id: 2bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02 + jinja: 'If the English version says: {{text["en"]}}; then the French version + should say: + ||| {{text["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: version-en-fr-source+target + reference: '' + 73dc1b77-e8ea-4dc8-8a12-0abc3b0dbba0: !Template + answer_choices: null + id: 73dc1b77-e8ea-4dc8-8a12-0abc3b0dbba0 + jinja: 'Given the following source text in French: {{text["fr"]}} , a good + English translation is: ||| {{text["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: a_good_translation-fr-en-source+target + 63dc1b77-e8ea-4dc8-8a12-0abc3b0dbba0: !Template + answer_choices: null + id: 63dc1b77-e8ea-4dc8-8a12-0abc3b0dbba0 + jinja: 'Given the following source text in English: {{text["en"]}} , a good + French translation is: ||| {{text["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: a_good_translation-en-fr-source+target + 3bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02: !Template + answer_choices: null + id: 3bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02 + jinja: 'Document in English: {{text["en"]}}\n\nTranslate the previous document to proper French: + ||| {{text["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: prev_doc-en-fr + reference: '' + 5bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02: !Template + answer_choices: null + id: 5bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02 + jinja: 'Document in French: {{text["fr"]}}\n\nTranslate the previous document to proper English: + ||| {{text["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: prev_doc-fr-en + reference: '' From a65e07af97a19c09679c58266e18277c866429d7 Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Wed, 17 Aug 2022 21:14:38 +0200 Subject: [PATCH 3/6] Add long prompts --- .../multi_eurlex/all_languages/templates.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/promptsource/templates/multi_eurlex/all_languages/templates.yaml b/promptsource/templates/multi_eurlex/all_languages/templates.yaml index 11dfaa26a..d949cf25b 100644 --- a/promptsource/templates/multi_eurlex/all_languages/templates.yaml +++ b/promptsource/templates/multi_eurlex/all_languages/templates.yaml @@ -79,3 +79,16 @@ templates: original_task: true name: prev_doc-fr-en reference: '' + 9bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02: !Template + answer_choices: null + id: 9bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02 + jinja: 'Document in French: {{text["fr"]}}\n\nTranslate the entire previous document to proper English sentence for sentence (min 100 words): + ||| {{text["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: prev_doc_long-fr-en + reference: '' From 7d38102270154c2411df58fda6af44438505176d Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Wed, 24 Aug 2022 16:01:03 +0200 Subject: [PATCH 4/6] Add ref --- promptsource/templates/multi_eurlex/all_languages/templates.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/promptsource/templates/multi_eurlex/all_languages/templates.yaml b/promptsource/templates/multi_eurlex/all_languages/templates.yaml index d949cf25b..85cf78dc6 100644 --- a/promptsource/templates/multi_eurlex/all_languages/templates.yaml +++ b/promptsource/templates/multi_eurlex/all_languages/templates.yaml @@ -41,6 +41,7 @@ templates: - BLEU original_task: true name: a_good_translation-fr-en-source+target + reference: '' 63dc1b77-e8ea-4dc8-8a12-0abc3b0dbba0: !Template answer_choices: null id: 63dc1b77-e8ea-4dc8-8a12-0abc3b0dbba0 From 738b303837c029fb075ed6f20000128a2d14bda6 Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Fri, 2 Sep 2022 22:15:35 +0200 Subject: [PATCH 5/6] Add prompts --- .../multi_eurlex/all_languages/templates.yaml | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) diff --git a/promptsource/templates/multi_eurlex/all_languages/templates.yaml b/promptsource/templates/multi_eurlex/all_languages/templates.yaml index 85cf78dc6..65754b8ee 100644 --- a/promptsource/templates/multi_eurlex/all_languages/templates.yaml +++ b/promptsource/templates/multi_eurlex/all_languages/templates.yaml @@ -54,6 +54,7 @@ templates: - BLEU original_task: true name: a_good_translation-en-fr-source+target + reference: '' 3bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02: !Template answer_choices: null id: 3bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02 @@ -93,3 +94,190 @@ templates: original_task: true name: prev_doc_long-fr-en reference: '' + 49ddea55-1710-4615-bbfa-fe5803e21e43: !Template + answer_choices: null + id: 49ddea55-1710-4615-bbfa-fe5803e21e43 + jinja: 'The French version says: {{text["fr"]}}; Thus the Spanish version + should say: + ||| {{text["es"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: version-fr-es-source+target + reference: '' + 2bc1f46c-d1fe-4bc9-99d1-9b61aa42cd02: !Template + answer_choices: null + id: 2bc1f46c-d1fe-4bc9-99d1-9b61aa42cd02 + jinja: 'Spanish version says: {{text["es"]}}; hence the Portuguese version + should say: + ||| {{text["pt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: version-es-pt-source+target + reference: '' + 73dc1b99-e8ea-4dc8-8a12-0abc3b0dbba0: !Template + answer_choices: null + id: 73dc1b99-e8ea-4dc8-8a12-0abc3b0dbba0 + jinja: 'Given the following source text in Portuguese: {{text["pt"]}} , a good + Spanish translation is: ||| {{text["es"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: a_good_translation-pt-es-source+target + reference: '' + 63dc1b99-e8ea-4dc8-8a12-0abc3b0dbba0: !Template + answer_choices: null + id: 63dc1b99-e8ea-4dc8-8a12-0abc3b0dbba0 + jinja: 'Source text in English: {{text["en"]}} , a good + Portuguese translation is: ||| {{text["pt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: a_good_translation-en-pt-source+target + reference: '' + 3bc0e88c-d1fe-4bc9-99d1-9b61aa42cd02: !Template + answer_choices: null + id: 3bc0e88c-d1fe-4bc9-99d1-9b61aa42cd02 + jinja: 'Document in Spanish: {{text["es"]}}\n\nTranslate the previous document to proper French: + ||| {{text["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: prev_doc-es-fr + reference: '' + 5bc1f46c-d1fe-4bc9-99d1-9b61aa42cd02: !Template + answer_choices: null + id: 5bc1f46c-d1fe-4bc9-99d1-9b61aa42cd02 + jinja: 'Document in French: {{text["fr"]}}\n\nTranslate the previous document to proper Portuguese: + ||| {{text["pt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: prev_doc-fr-pt + reference: '' + 9bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02: !Template + answer_choices: null + id: 9bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02 + jinja: 'Document in the language of Portugal: {{text["pt"]}}\n\nTranslate the entire previous document to proper English sentence for sentence (min 100 words): + ||| {{text["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: prev_doc_long-pt-en + reference: '' + 49ddee85-1710-4615-bbfa-fe5803e21e43: !Template + answer_choices: null + id: 49ddee85-1710-4615-bbfa-fe5803e21e43 + jinja: 'The English version says: {{text["en"]}}; Thus in Spanish the version + should say: + ||| {{text["es"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: version-en-es-source+target + reference: '' + 2bc1f46c-d1fe-4bc9-99d1-9b61aa42cd02: !Template + answer_choices: null + id: 2bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02 + jinja: 'Portuguese version says: {{text["pt"]}}; hence the French version + should say: + ||| {{text["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: version-pt-fr-source+target + reference: '' + 73dc1b99-e8ea-4dc8-8a12-0abc3b0dbba0: !Template + answer_choices: null + id: 73dc1b99-e8ea-4dc8-8a12-0abc3b0dbba0 + jinja: 'Given this in French: {{text["fr"]}} ... a good + Portuguese translation is: ||| {{text["pt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: a_good_translation-fr-pt-source+target + reference: '' + 63dc1b99-e8eb-4dc8-8a12-0abc3b0dbba0: !Template + answer_choices: null + id: 63dc1b77-e8eb-4dc8-8a12-0abc3b0dbba0 + jinja: 'Source text in English: {{text["en"]}} , a good + Spanish translation is: ||| {{text["es"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: a_good_translation-en-pt-source+target + reference: '' + 3bc0e99c-d1fe-4bc9-99d1-9b61aa42cd02: !Template + answer_choices: null + id: 3bc0e99c-d1fe-4bc9-99d1-9b61aa42cd02 + jinja: 'Document in Spanish: {{text["es"]}}\n\nTranslate the previous document to proper English: + ||| {{text["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: prev_doc-es-en + reference: '' + 5bc1f55c-d1fe-4bc9-99d1-9b61aa42cd02: !Template + answer_choices: null + id: 5bc1f55c-d1fe-4bc9-99d1-9b61aa42cd02 + jinja: 'Document in Portuguese: {{text["fr"]}}\n\nTranslate the previous document to proper English: + ||| {{text["pt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: prev_doc-pt-en + reference: '' + 9bc0e73c-d1fe-4bc9-99d1-9b61aa42cd02: !Template + answer_choices: null + id: 9bc0e73c-d1fe-4bc9-99d1-9b61aa42cd02 + jinja: 'Document in English: {{text["pt"]}}\n\nTranslate the entire previous document to the most common language in Spain sentence for sentence (min 100 words): + ||| {{text["es"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: + - BLEU + original_task: true + name: prev_doc_long-en-es + reference: '' + From c70ecd05f07b58b3257382b6a7c303ae188c2ff5 Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Sun, 4 Sep 2022 23:18:08 +0200 Subject: [PATCH 6/6] Fix uuid --- .../templates/multi_eurlex/all_languages/templates.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/promptsource/templates/multi_eurlex/all_languages/templates.yaml b/promptsource/templates/multi_eurlex/all_languages/templates.yaml index 65754b8ee..80ae83d2b 100644 --- a/promptsource/templates/multi_eurlex/all_languages/templates.yaml +++ b/promptsource/templates/multi_eurlex/all_languages/templates.yaml @@ -122,9 +122,9 @@ templates: original_task: true name: version-es-pt-source+target reference: '' - 73dc1b99-e8ea-4dc8-8a12-0abc3b0dbba0: !Template + 74ec1b99-e8ea-4dc8-8a12-0abc3b0dbba0: !Template answer_choices: null - id: 73dc1b99-e8ea-4dc8-8a12-0abc3b0dbba0 + id: 74ec1b99-e8ea-4dc8-8a12-0abc3b0dbba0 jinja: 'Given the following source text in Portuguese: {{text["pt"]}} , a good Spanish translation is: ||| {{text["es"]}}' metadata: !TemplateMetadata @@ -201,9 +201,9 @@ templates: original_task: true name: version-en-es-source+target reference: '' - 2bc1f46c-d1fe-4bc9-99d1-9b61aa42cd02: !Template + 2bc2e46c-d1fe-4bc9-99d1-9b61aa42cd02: !Template answer_choices: null - id: 2bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02 + id: 2bc2e46c-d1fe-4bc9-99d1-9b61aa42cd02 jinja: 'Portuguese version says: {{text["pt"]}}; hence the French version should say: ||| {{text["fr"]}}'