From da5e046419e22bb4f0a70aeac7bf3cbb1ba2e3e1 Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Fri, 29 Apr 2022 01:16:08 +0200
Subject: [PATCH 01/21] added first templates for DiaBLa dataset rbawden/DiaBLa

---
 promptsource/templates.py                     |   2 +-
 .../templates/rbawden/DiaBLa/templates.yaml   | 476 ++++++++++++++++++
 2 files changed, 477 insertions(+), 1 deletion(-)
 create mode 100644 promptsource/templates/rbawden/DiaBLa/templates.yaml

diff --git a/promptsource/templates.py b/promptsource/templates.py
index 3ece927fe..fe05949c2 100644
--- a/promptsource/templates.py
+++ b/promptsource/templates.py
@@ -26,7 +26,7 @@
 
 # These are users whose datasets should be included in the results returned by
 # filter_english_datasets (regardless of their metadata)
-INCLUDED_USERS = {"Zaid", "craffel"}
+INCLUDED_USERS = {"Zaid", "craffel", "rbawden"}
 
 
 def highlight(input):
diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
new file mode 100644
index 000000000..048675357
--- /dev/null
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -0,0 +1,476 @@
+dataset: rbawden/DiaBLa
+templates:
+  0e55b858-92b0-4e5b-923b-0d23518035e3: !Template
+    answer_choices: null
+    id: 0e55b858-92b0-4e5b-923b-0d23518035e3
+    jinja: '{% if utterance_meta.lang == "french" %}
+
+      {% if dialogue_history|length > 0 %}
+
+
+      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
+      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
+      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
+      %}{% endif %}" translates into English as: {{ previous_ref }}
+
+
+      "{{ orig }}" translates into English as: ||| {{ ref }}
+
+
+      {% endif %}
+
+      {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_1-context-analogy_lang-given_fr2en
+    reference: 1 previous context (same language) used for analogy, French-to-English
+  0e841022-eb3c-4f5a-b57a-0cbd3d628b68: !Template
+    answer_choices: null
+    id: 0e841022-eb3c-4f5a-b57a-0cbd3d628b68
+    jinja: "{% if utterance_meta.lang == \"english\" %}\n{% if dialogue_history|length\
+      \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
+      \ set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous\
+      \ in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang\
+      \ %}A{% else %}B{% endif %}: {{ previous.orig }}\n{% endfor %} \nTranslate the\
+      \ next utterance in the dialogue into {% if utterance_meta.lang == \"english\"\
+      \ %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang == first_lang\
+      \ %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang ==\
+      \ first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif %}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_5-context_orig-lang_en2fr
+    reference: Up to 5 previous sentences (original language), English-to-French
+  123f36ee-c808-4cf0-aa5c-ed1f03ac3b69: !Template
+    answer_choices: yes || no
+    id: 123f36ee-c808-4cf0-aa5c-ed1f03ac3b69
+    jinja: "{% if utterance_meta.lang == \"english\" %}\n{% set options = [\"word\
+      \ choice\", \"grammar\", \"style\", \"coherence\", \"meaning\"] %}\n{% set label\
+      \ = range(0,5)|random %}\n{% if options[label] in utterance_meta.eval_problems\
+      \ %}{% set reply=0 %}{% else %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
+      \ > 0 %}\n{% if dialogue_history|length > 0 %}\nGiven the following dialogue\
+      \ between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
+      \ != utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
+      \ endif %}{% endfor %} \n{% if utterance_meta.lang == first_lang %}A{% else\
+      \ %}B{% endif %}: {{ mt }}\n\nDoes the last utterance contain a {{ options[label]\
+      \ }} problem, {{ \"yes\" }} or {{ \"no\" }}?\n\n||| {{ [\"yes\", \"no\" ][reply]\
+      \ }}\n{% endif %}\n{% endif %}\n{% endif %}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
+      original_task: false
+    name: classify-errors_same-lang_fr
+    reference: Identify presence of notable errors, same language context (5), French
+      translation
+  1798dee9-8a9c-45ed-87d8-237ec7dbf5ca: !Template
+    answer_choices: null
+    id: 1798dee9-8a9c-45ed-87d8-237ec7dbf5ca
+    jinja: "{% if utterance_meta.lang == \"english\" %}\n{% if dialogue_history|length\
+      \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
+      \ set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous\
+      \ in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang\
+      \ %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang\
+      \ %}{{ previous.orig }}{% else %}{{ previous.ref }}{% endif %}\n{% endfor %}\
+      \ \nTranslate the next utterance in the dialogue into {% if utterance_meta.lang\
+      \ == \"english\" %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif\
+      \ %}\n"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_5-context_same-lang-ref_en2fr
+    reference: Up to 5 previous sentences (same language, ref), English-to-French
+  2731216a-b994-48f9-aaf6-00c7038bbed5: !Template
+    answer_choices: null
+    id: 2731216a-b994-48f9-aaf6-00c7038bbed5
+    jinja: "{% if dialogue_history|length > 0 %}\nGiven the following dialogue between\
+      \ person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
+      \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.ref }}{%\
+      \ endif %}\n{% endfor %} \nTranslate the next utterance in the dialogue into\
+      \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
+      \ %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{\
+      \ orig }}\n\n||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
+      \ %}: {{ ref }}\n{% endif %}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_5-context_same-lang-ref_both-directions
+    reference: Up to 5 previous sentences (same language, ref), both language directions
+  28ea04f4-338e-40cf-8730-4a794b5b64b2: !Template
+    answer_choices: yes || no
+    id: 28ea04f4-338e-40cf-8730-4a794b5b64b2
+    jinja: "{% set options = [\"word choice\", \"grammar\", \"style\", \"coherence\"\
+      , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% if options[label]\
+      \ in utterance_meta.eval_problems %}{% set reply=0 %}{% else %}{% set reply=1\
+      \ %}{% endif %}\n{% if utterance_meta.eval_problems|length > 0 %}\n{% if dialogue_history|length\
+      \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
+      \ set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous\
+      \ in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang\
+      \ %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang != utterance_meta.lang\
+      \ %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}{% endfor %} \n\
+      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ mt }}\n\
+      \nDoes the last utterance contain a {{ options[label] }} problem, {{ \"yes\"\
+      \ }} or {{ \"no\" }}?\n\n||| {{ [\"yes\", \"no\" ][reply] }}\n{% endif %}\n\
+      {% endif %}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
+      original_task: false
+    name: classify-errors_same-lang_both-directions
+    reference: Identify presence of notable errors
+  2abc728f-e7fc-44fc-ba2a-ad78ae2469a9: !Template
+    answer_choices: null
+    id: 2abc728f-e7fc-44fc-ba2a-ad78ae2469a9
+    jinja: '{% if utterance_meta.lang == "english" %}
+
+      {% if dialogue_history|length > 0 %}
+
+
+      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
+      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
+      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
+      %}{% endif %}" translates as: {{ previous_ref }}
+
+
+      "{{ orig }}" translates as: ||| {{ ref }}
+
+
+      {% endif %}
+
+      {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_1-context-analogy_infer-lang_en2fr
+    reference: 1 previous context (same language) used for analogy, English-to-French
+  40c1ee21-3cf4-4efc-b902-d3aab0a79c40: !Template
+    answer_choices: null
+    id: 40c1ee21-3cf4-4efc-b902-d3aab0a79c40
+    jinja: '{% if utterance_meta.lang == "french" %}
+
+      {% if dialogue_history|length > 0 %}
+
+
+      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
+      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
+      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
+      %}{% endif %}" translates as: {{ previous_ref }}
+
+
+      "{{ orig }}" translates as: ||| {{ ref }}
+
+
+      {% endif %}
+
+      {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_1-context-analogy_infer-lang_fr2en
+    reference: 1 previous context (same language) used for analogy, French-to-English
+  6a01fbe6-d5ec-4ad9-a2ee-3c48ed095885: !Template
+    answer_choices: null
+    id: 6a01fbe6-d5ec-4ad9-a2ee-3c48ed095885
+    jinja: '{% if dialogue_history|length > 0 %}
+
+      {% if utterance_meta.lang == "french" %}{% set other_lang = "English" %}{% else
+      %}{% set other_lang = "French" %}{% endif %}
+
+      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
+      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
+      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
+      %}{% endif %}" translates into {{ other_lang }} as: {{ previous_ref }}
+
+
+      "{{ orig }}" translates into {{ other_lang }} as: ||| {{ ref }}
+
+
+      {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_1-context-analogy_lang-given_both-directions
+    reference: 1 previous context (same language) used for analogy, both directions
+  702f1f50-d6b0-4e65-a8e6-b449257f1e35: !Template
+    answer_choices: null
+    id: 702f1f50-d6b0-4e65-a8e6-b449257f1e35
+    jinja: "{% if utterance_meta.lang == \"french\" %}\n{% if dialogue_history|length\
+      \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
+      \ set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous\
+      \ in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang\
+      \ %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang\
+      \ %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}\n{% endfor %}\
+      \ \nTranslate the next utterance in the dialogue into {% if utterance_meta.lang\
+      \ == \"english\" %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif\
+      \ %}\n"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_5-context_same-lang-ref_fr2en
+    reference: Up to 5 previous sentences (same language, ref), French-to-English
+  72759632-7764-4493-9a6e-8f1c8b337bde: !Template
+    answer_choices: null
+    id: 72759632-7764-4493-9a6e-8f1c8b337bde
+    jinja: "{% if utterance_meta.lang == \"french\" %}\n{% if dialogue_history|length\
+      \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
+      \ set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous\
+      \ in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang\
+      \ %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang\
+      \ %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}\n{% endfor %}\
+      \ \nTranslate the next utterance in the dialogue into {% if utterance_meta.lang\
+      \ == \"english\" %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif\
+      \ %}\n"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_5-context_same-lang-mt_fr2en
+    reference: Up to 5 previous sentences (same language, MT), French-to-English
+  72c1cbbd-956f-4721-9cee-fd50148cf77d: !Template
+    answer_choices: null
+    id: 72c1cbbd-956f-4721-9cee-fd50148cf77d
+    jinja: '{% if utterance_meta.lang == "french"  %}
+
+      Translate the following into English: "{{ orig }}" ||| {{ ref }}
+
+      {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_sent_fr2en
+    reference: Sentence-level, French-to-English
+  836c5096-2fd6-478c-89e8-39a050347d28: !Template
+    answer_choices: null
+    id: 836c5096-2fd6-478c-89e8-39a050347d28
+    jinja: "{% if utterance_meta.lang == \"french\" %}\n{% if dialogue_history|length\
+      \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
+      \ set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous\
+      \ in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang\
+      \ %}A{% else %}B{% endif %}: {{ previous.orig }}\n{% endfor %} \nTranslate the\
+      \ next utterance in the dialogue into {% if utterance_meta.lang == \"english\"\
+      \ %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang == first_lang\
+      \ %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang ==\
+      \ first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif %}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: false
+    name: MT_5-context_orig-lang_fr2en
+    reference: Up to 5 previous sentences (original language), French-to-English
+  842dc41a-8af0-4dca-8b55-a87026bfac31: !Template
+    answer_choices: null
+    id: 842dc41a-8af0-4dca-8b55-a87026bfac31
+    jinja: 'Translate the following into {% if utterance_meta.lang == "english"  %}French{%
+      else %}English{% endif %}: "{{ orig }}"? ||| {{ ref }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_sent_both-directions
+    reference: Sentence-level, both directions
+  93f5256d-bd93-4056-b466-152b55860d02: !Template
+    answer_choices: null
+    id: 93f5256d-bd93-4056-b466-152b55860d02
+    jinja: "{% if dialogue_history|length > 0 %}\nGiven the following dialogue between\
+      \ person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ previous.orig }}\n{% endfor %}\
+      \ \nTranslate the next utterance in the dialogue into {% if utterance_meta.lang\
+      \ == \"english\" %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_5-context_orig-lang_both-directions
+    reference: Up to 5 previous sentences (original language), both language directions
+  96da9c0e-e606-4283-8494-7129c75b4d02: !Template
+    answer_choices: null
+    id: 96da9c0e-e606-4283-8494-7129c75b4d02
+    jinja: '{% if utterance_meta.lang == "english"  %}
+
+      Translate the following into French: "{{ orig }}" ||| {{ ref }}
+
+      {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_sent_en2fr
+    reference: Sentence-level, English-to-French
+  97d56019-a7df-4733-aed0-332c222e9079: !Template
+    answer_choices: null
+    id: 97d56019-a7df-4733-aed0-332c222e9079
+    jinja: '{% if utterance_meta.lang == "english" %}
+
+      {% if dialogue_history|length > 0 %}
+
+
+      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
+      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
+      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
+      %}{% endif %}" translates into French as: {{ previous_ref }}
+
+
+      "{{ orig }}" translates into French as: ||| {{ ref }}
+
+
+      {% endif %}
+
+      {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_1-context-analogy_lang-given_en2fr
+    reference: 1 previous context (same language) used for analogy, English-to-French
+  ac4c63da-32d2-40ac-aa7a-632e8ba42b4a: !Template
+    answer_choices: A ||| B
+    id: ac4c63da-32d2-40ac-aa7a-632e8ba42b4a
+    jinja: '{% if ref != mt %}
+
+      Which of the following translations of "{{ orig }}" is produced automatically?
+
+      {% set label = range(0,2)|random %}
+
+      {{ "A" }}) {% if label == 0 %}{{ ref }}{% else %}{{ mt }}{% endif %}
+
+      {{ "B" }}) {% if label == 0 %}{{ mt }}{% else %}{{ ref }}{% endif %}
+
+      ||| {{ ["A", "B"][label] }}
+
+      {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
+      original_task: false
+    name: discriminate-mt-ref
+    reference: Identify-MT-output
+  b61c81ec-29eb-47f8-a1c6-561264ac04f3: !Template
+    answer_choices: null
+    id: b61c81ec-29eb-47f8-a1c6-561264ac04f3
+    jinja: "{% if dialogue_history|length > 0 %}\nGiven the following dialogue between\
+      \ person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
+      \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
+      \ endif %}\n{% endfor %} \nTranslate the next utterance in the dialogue into\
+      \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
+      \ %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{\
+      \ orig }}\n\n||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
+      \ %}: {{ ref }}\n{% endif %}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_5-context_same-lang-mt_both-directions
+    reference: Up to 5 previous sentences (same language, MT), both language directions
+  d068ba77-ce75-4de3-a170-ca0a7cd3f8d1: !Template
+    answer_choices: null
+    id: d068ba77-ce75-4de3-a170-ca0a7cd3f8d1
+    jinja: "{% if utterance_meta.lang == \"english\" %}\n{% if dialogue_history|length\
+      \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
+      \ set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous\
+      \ in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang\
+      \ %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang\
+      \ %}{{ previous.orig }}{% else %}{{ previous.ref }}{% endif %}\n{% endfor %}\
+      \ \nTranslate the next utterance in the dialogue into {% if utterance_meta.lang\
+      \ == \"english\" %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif\
+      \ %}\n"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_5-context_same-lang-mt_en2fr
+    reference: Up to 5 previous sentences (same language, MT), English-to-French
+  dddd6f35-5a01-431b-b59b-e9a8e76377aa: !Template
+    answer_choices: yes || no
+    id: dddd6f35-5a01-431b-b59b-e9a8e76377aa
+    jinja: "{% if utterance_meta.lang == \"french\" %}\n{% set options = [\"word choice\"\
+      , \"grammar\", \"style\", \"coherence\", \"meaning\"] %}\n{% set label = range(0,5)|random\
+      \ %}\n{% if options[label] in utterance_meta.eval_problems %}{% set reply=0\
+      \ %}{% else %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
+      \ > 0 %}\n{% if dialogue_history|length > 0 %}\nGiven the following dialogue\
+      \ between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
+      \ != utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
+      \ endif %}{% endfor %} \n{% if utterance_meta.lang == first_lang %}A{% else\
+      \ %}B{% endif %}: {{ mt }}\n\nDoes the last utterance contain a {{ options[label]\
+      \ }} problem, {{ \"yes\" }} or {{ \"no\" }}?\n\n||| {{ [\"yes\", \"no\" ][reply]\
+      \ }}\n{% endif %}\n{% endif %}\n{% endif %}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
+      original_task: false
+    name: classify-errors_same-lang_en
+    reference: Identify presence of notable errors, same language context (5), English
+      translation
+  eea8f47e-9bf5-4423-980b-58a9635c1f49: !Template
+    answer_choices: 1 previous context (same language) used for analogy, both directions
+    id: eea8f47e-9bf5-4423-980b-58a9635c1f49
+    jinja: '{% if dialogue_history|length > 0 %}
+
+      {% if utterance_meta.lang == "french" %}{% set other_lang = "English" %}{% else
+      %}{% set other_lang = "French" %}{% endif %}
+
+      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
+      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
+      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
+      %}{% endif %}" translates as: {{ previous_ref }}
+
+
+      "{{ orig }}" translates as: ||| {{ ref }}
+
+
+      {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT_1-context-analogy_infer-lang_both-directions
+    reference: MT task, 1 previous context (orig language)

From 77ba6e9f814b5bb54c1016171597627c2b8e845a Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Fri, 29 Apr 2022 01:33:16 +0200
Subject: [PATCH 02/21] declared previous_ref at beginning of templates

---
 .../templates/rbawden/DiaBLa/templates.yaml   | 25 +++++++++++--------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index 048675357..bd7e905eb 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -3,7 +3,9 @@ templates:
   0e55b858-92b0-4e5b-923b-0d23518035e3: !Template
     answer_choices: null
     id: 0e55b858-92b0-4e5b-923b-0d23518035e3
-    jinja: '{% if utterance_meta.lang == "french" %}
+    jinja: '{% set previous_ref = "" %}
+
+      {% if utterance_meta.lang == "french" %}
 
       {% if dialogue_history|length > 0 %}
 
@@ -137,11 +139,10 @@ templates:
   2abc728f-e7fc-44fc-ba2a-ad78ae2469a9: !Template
     answer_choices: null
     id: 2abc728f-e7fc-44fc-ba2a-ad78ae2469a9
-    jinja: '{% if utterance_meta.lang == "english" %}
+    jinja: '{% set previous_ref = "" %}{% if utterance_meta.lang == "english" %}
 
       {% if dialogue_history|length > 0 %}
 
-
       "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
       dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
       else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
@@ -164,11 +165,10 @@ templates:
   40c1ee21-3cf4-4efc-b902-d3aab0a79c40: !Template
     answer_choices: null
     id: 40c1ee21-3cf4-4efc-b902-d3aab0a79c40
-    jinja: '{% if utterance_meta.lang == "french" %}
+    jinja: '{% set previous_ref = "" %}{% if utterance_meta.lang == "french" %}
 
       {% if dialogue_history|length > 0 %}
 
-
       "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
       dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
       else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
@@ -191,10 +191,11 @@ templates:
   6a01fbe6-d5ec-4ad9-a2ee-3c48ed095885: !Template
     answer_choices: null
     id: 6a01fbe6-d5ec-4ad9-a2ee-3c48ed095885
-    jinja: '{% if dialogue_history|length > 0 %}
+    jinja: '{% set previous_ref = "" %}
 
-      {% if utterance_meta.lang == "french" %}{% set other_lang = "English" %}{% else
-      %}{% set other_lang = "French" %}{% endif %}
+      {% if dialogue_history|length > 0 %}{% if utterance_meta.lang == "french" %}{%
+      set other_lang = "English" %}{% else %}{% set other_lang = "French" %}{% endif
+      %}
 
       "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
       dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
@@ -337,7 +338,9 @@ templates:
   97d56019-a7df-4733-aed0-332c222e9079: !Template
     answer_choices: null
     id: 97d56019-a7df-4733-aed0-332c222e9079
-    jinja: '{% if utterance_meta.lang == "english" %}
+    jinja: '{% set previous_ref = "" %}
+
+      {% if utterance_meta.lang == "english" %}
 
       {% if dialogue_history|length > 0 %}
 
@@ -452,7 +455,9 @@ templates:
   eea8f47e-9bf5-4423-980b-58a9635c1f49: !Template
     answer_choices: 1 previous context (same language) used for analogy, both directions
     id: eea8f47e-9bf5-4423-980b-58a9635c1f49
-    jinja: '{% if dialogue_history|length > 0 %}
+    jinja: '{% set previous_ref = "" %}
+
+      {% if dialogue_history|length > 0 %}
 
       {% if utterance_meta.lang == "french" %}{% set other_lang = "English" %}{% else
       %}{% set other_lang = "French" %}{% endif %}

From fa96aa0a5d41abc6306babffd62643d4569d1392 Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Fri, 29 Apr 2022 01:45:08 +0200
Subject: [PATCH 03/21] declared more variables at beginning of templates

---
 .../templates/rbawden/DiaBLa/templates.yaml   | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index bd7e905eb..c9869b2a8 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -51,10 +51,10 @@ templates:
   123f36ee-c808-4cf0-aa5c-ed1f03ac3b69: !Template
     answer_choices: yes || no
     id: 123f36ee-c808-4cf0-aa5c-ed1f03ac3b69
-    jinja: "{% if utterance_meta.lang == \"english\" %}\n{% set options = [\"word\
-      \ choice\", \"grammar\", \"style\", \"coherence\", \"meaning\"] %}\n{% set label\
-      \ = range(0,5)|random %}\n{% if options[label] in utterance_meta.eval_problems\
-      \ %}{% set reply=0 %}{% else %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
+    jinja: "{% set options = [\"word choice\", \"grammar\", \"style\", \"coherence\"\
+      , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% if utterance_meta.lang\
+      \ == \"english\" %}\n{% if options[label] in utterance_meta.eval_problems %}{%\
+      \ set reply=0 %}{% else %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
       \ > 0 %}\n{% if dialogue_history|length > 0 %}\nGiven the following dialogue\
       \ between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
@@ -191,7 +191,7 @@ templates:
   6a01fbe6-d5ec-4ad9-a2ee-3c48ed095885: !Template
     answer_choices: null
     id: 6a01fbe6-d5ec-4ad9-a2ee-3c48ed095885
-    jinja: '{% set previous_ref = "" %}
+    jinja: '{% set previous_ref = "" %}{% set other_lang = "" %}
 
       {% if dialogue_history|length > 0 %}{% if utterance_meta.lang == "french" %}{%
       set other_lang = "English" %}{% else %}{% set other_lang = "French" %}{% endif
@@ -367,11 +367,11 @@ templates:
   ac4c63da-32d2-40ac-aa7a-632e8ba42b4a: !Template
     answer_choices: A ||| B
     id: ac4c63da-32d2-40ac-aa7a-632e8ba42b4a
-    jinja: '{% if ref != mt %}
+    jinja: '{% set label = range(0,2)|random %}
 
-      Which of the following translations of "{{ orig }}" is produced automatically?
+      {% if ref != mt %}
 
-      {% set label = range(0,2)|random %}
+      Which of the following translations of "{{ orig }}" is produced automatically?
 
       {{ "A" }}) {% if label == 0 %}{{ ref }}{% else %}{{ mt }}{% endif %}
 
@@ -431,10 +431,10 @@ templates:
   dddd6f35-5a01-431b-b59b-e9a8e76377aa: !Template
     answer_choices: yes || no
     id: dddd6f35-5a01-431b-b59b-e9a8e76377aa
-    jinja: "{% if utterance_meta.lang == \"french\" %}\n{% set options = [\"word choice\"\
-      , \"grammar\", \"style\", \"coherence\", \"meaning\"] %}\n{% set label = range(0,5)|random\
-      \ %}\n{% if options[label] in utterance_meta.eval_problems %}{% set reply=0\
-      \ %}{% else %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
+    jinja: "{% set options = [\"word choice\", \"grammar\", \"style\", \"coherence\"\
+      , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% if utterance_meta.lang\
+      \ == \"french\" %}\n{% if options[label] in utterance_meta.eval_problems %}{%\
+      \ set reply=0 %}{% else %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
       \ > 0 %}\n{% if dialogue_history|length > 0 %}\nGiven the following dialogue\
       \ between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
@@ -455,7 +455,7 @@ templates:
   eea8f47e-9bf5-4423-980b-58a9635c1f49: !Template
     answer_choices: 1 previous context (same language) used for analogy, both directions
     id: eea8f47e-9bf5-4423-980b-58a9635c1f49
-    jinja: '{% set previous_ref = "" %}
+    jinja: '{% set previous_ref = "" %}{% set other_lang = "" %}
 
       {% if dialogue_history|length > 0 %}
 

From 3a19cfb1df609292bfd1698432ca943e2ab53118 Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Fri, 29 Apr 2022 01:54:55 +0200
Subject: [PATCH 04/21] declared more variables at beginning of templates

---
 .../templates/rbawden/DiaBLa/templates.yaml   | 152 +++++++++---------
 1 file changed, 77 insertions(+), 75 deletions(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index c9869b2a8..6f5afac93 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -32,15 +32,16 @@ templates:
   0e841022-eb3c-4f5a-b57a-0cbd3d628b68: !Template
     answer_choices: null
     id: 0e841022-eb3c-4f5a-b57a-0cbd3d628b68
-    jinja: "{% if utterance_meta.lang == \"english\" %}\n{% if dialogue_history|length\
-      \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
-      \ set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous\
-      \ in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang\
-      \ %}A{% else %}B{% endif %}: {{ previous.orig }}\n{% endfor %} \nTranslate the\
-      \ next utterance in the dialogue into {% if utterance_meta.lang == \"english\"\
-      \ %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang == first_lang\
-      \ %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang ==\
-      \ first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif %}"
+    jinja: "{% set first_lang=\"\" %}\n{% if utterance_meta.lang == \"english\" %}\n\
+      {% if dialogue_history|length > 0 %}\nGiven the following dialogue between person\
+      \ A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ previous.orig }}\n{% endfor %}\
+      \ \nTranslate the next utterance in the dialogue into {% if utterance_meta.lang\
+      \ == \"english\" %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif\
+      \ %}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -52,9 +53,9 @@ templates:
     answer_choices: yes || no
     id: 123f36ee-c808-4cf0-aa5c-ed1f03ac3b69
     jinja: "{% set options = [\"word choice\", \"grammar\", \"style\", \"coherence\"\
-      , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% if utterance_meta.lang\
-      \ == \"english\" %}\n{% if options[label] in utterance_meta.eval_problems %}{%\
-      \ set reply=0 %}{% else %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
+      , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% set reply=0 %}\n\
+      {% if utterance_meta.lang == \"english\" %}\n{% if options[label] in utterance_meta.eval_problems\
+      \ %}{% set reply=0 %}{% else %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
       \ > 0 %}\n{% if dialogue_history|length > 0 %}\nGiven the following dialogue\
       \ between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
@@ -96,8 +97,8 @@ templates:
   2731216a-b994-48f9-aaf6-00c7038bbed5: !Template
     answer_choices: null
     id: 2731216a-b994-48f9-aaf6-00c7038bbed5
-    jinja: "{% if dialogue_history|length > 0 %}\nGiven the following dialogue between\
-      \ person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+    jinja: "{% set first_lang=\"\" %}\n{% if dialogue_history|length > 0 %}\nGiven\
+      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
       \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
       \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.ref }}{%\
@@ -117,18 +118,18 @@ templates:
     answer_choices: yes || no
     id: 28ea04f4-338e-40cf-8730-4a794b5b64b2
     jinja: "{% set options = [\"word choice\", \"grammar\", \"style\", \"coherence\"\
-      , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% if options[label]\
-      \ in utterance_meta.eval_problems %}{% set reply=0 %}{% else %}{% set reply=1\
-      \ %}{% endif %}\n{% if utterance_meta.eval_problems|length > 0 %}\n{% if dialogue_history|length\
-      \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
-      \ set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous\
-      \ in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang\
-      \ %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang != utterance_meta.lang\
-      \ %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}{% endfor %} \n\
-      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ mt }}\n\
-      \nDoes the last utterance contain a {{ options[label] }} problem, {{ \"yes\"\
-      \ }} or {{ \"no\" }}?\n\n||| {{ [\"yes\", \"no\" ][reply] }}\n{% endif %}\n\
-      {% endif %}"
+      , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% set reply=0 %}\n\
+      {% if options[label] in utterance_meta.eval_problems %}{% set reply=0 %}{% else\
+      \ %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
+      \ > 0 %}\n{% if dialogue_history|length > 0 %}\nGiven the following dialogue\
+      \ between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
+      \ != utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
+      \ endif %}{% endfor %} \n{% if utterance_meta.lang == first_lang %}A{% else\
+      \ %}B{% endif %}: {{ mt }}\n\nDoes the last utterance contain a {{ options[label]\
+      \ }} problem, {{ \"yes\" }} or {{ \"no\" }}?\n\n||| {{ [\"yes\", \"no\" ][reply]\
+      \ }}\n{% endif %}\n{% endif %}"
     metadata: !TemplateMetadata
       choices_in_prompt: true
       metrics:
@@ -217,17 +218,17 @@ templates:
   702f1f50-d6b0-4e65-a8e6-b449257f1e35: !Template
     answer_choices: null
     id: 702f1f50-d6b0-4e65-a8e6-b449257f1e35
-    jinja: "{% if utterance_meta.lang == \"french\" %}\n{% if dialogue_history|length\
-      \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
-      \ set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous\
-      \ in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang\
-      \ %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang\
-      \ %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}\n{% endfor %}\
-      \ \nTranslate the next utterance in the dialogue into {% if utterance_meta.lang\
-      \ == \"english\" %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif\
-      \ %}\n"
+    jinja: "{% set first_lang=\"\" %}\n{% if utterance_meta.lang == \"french\" %}\n\
+      {% if dialogue_history|length > 0 %}\nGiven the following dialogue between person\
+      \ A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
+      \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
+      \ endif %}\n{% endfor %} \nTranslate the next utterance in the dialogue into\
+      \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
+      \ %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{\
+      \ orig }}\n\n||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
+      \ %}: {{ ref }}\n{% endif %}\n{% endif %}\n"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -238,17 +239,17 @@ templates:
   72759632-7764-4493-9a6e-8f1c8b337bde: !Template
     answer_choices: null
     id: 72759632-7764-4493-9a6e-8f1c8b337bde
-    jinja: "{% if utterance_meta.lang == \"french\" %}\n{% if dialogue_history|length\
-      \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
-      \ set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous\
-      \ in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang\
-      \ %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang\
-      \ %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}\n{% endfor %}\
-      \ \nTranslate the next utterance in the dialogue into {% if utterance_meta.lang\
-      \ == \"english\" %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif\
-      \ %}\n"
+    jinja: "{% set first_lang=\"\" %}\n{% if utterance_meta.lang == \"french\" %}\n\
+      {% if dialogue_history|length > 0 %}\nGiven the following dialogue between person\
+      \ A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
+      \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
+      \ endif %}\n{% endfor %} \nTranslate the next utterance in the dialogue into\
+      \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
+      \ %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{\
+      \ orig }}\n\n||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
+      \ %}: {{ ref }}\n{% endif %}\n{% endif %}\n"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -274,15 +275,16 @@ templates:
   836c5096-2fd6-478c-89e8-39a050347d28: !Template
     answer_choices: null
     id: 836c5096-2fd6-478c-89e8-39a050347d28
-    jinja: "{% if utterance_meta.lang == \"french\" %}\n{% if dialogue_history|length\
-      \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
-      \ set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous\
-      \ in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang\
-      \ %}A{% else %}B{% endif %}: {{ previous.orig }}\n{% endfor %} \nTranslate the\
-      \ next utterance in the dialogue into {% if utterance_meta.lang == \"english\"\
-      \ %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang == first_lang\
-      \ %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang ==\
-      \ first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif %}"
+    jinja: "{% set first_lang=\"\" %}\n{% if utterance_meta.lang == \"french\" %}\n\
+      {% if dialogue_history|length > 0 %}\nGiven the following dialogue between person\
+      \ A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ previous.orig }}\n{% endfor %}\
+      \ \nTranslate the next utterance in the dialogue into {% if utterance_meta.lang\
+      \ == \"english\" %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif\
+      \ %}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -305,8 +307,8 @@ templates:
   93f5256d-bd93-4056-b466-152b55860d02: !Template
     answer_choices: null
     id: 93f5256d-bd93-4056-b466-152b55860d02
-    jinja: "{% if dialogue_history|length > 0 %}\nGiven the following dialogue between\
-      \ person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+    jinja: "{% set first_lang=\"\" %}\n{% if dialogue_history|length > 0 %}\nGiven\
+      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
       \ == first_lang %}A{% else %}B{% endif %}: {{ previous.orig }}\n{% endfor %}\
       \ \nTranslate the next utterance in the dialogue into {% if utterance_meta.lang\
@@ -390,8 +392,8 @@ templates:
   b61c81ec-29eb-47f8-a1c6-561264ac04f3: !Template
     answer_choices: null
     id: b61c81ec-29eb-47f8-a1c6-561264ac04f3
-    jinja: "{% if dialogue_history|length > 0 %}\nGiven the following dialogue between\
-      \ person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+    jinja: "{% set first_lang=\"\" %}\n{% if dialogue_history|length > 0 %}\nGiven\
+      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
       \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
       \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
@@ -410,17 +412,17 @@ templates:
   d068ba77-ce75-4de3-a170-ca0a7cd3f8d1: !Template
     answer_choices: null
     id: d068ba77-ce75-4de3-a170-ca0a7cd3f8d1
-    jinja: "{% if utterance_meta.lang == \"english\" %}\n{% if dialogue_history|length\
-      \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
-      \ set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous\
-      \ in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang\
-      \ %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang\
-      \ %}{{ previous.orig }}{% else %}{{ previous.ref }}{% endif %}\n{% endfor %}\
-      \ \nTranslate the next utterance in the dialogue into {% if utterance_meta.lang\
-      \ == \"english\" %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif\
-      \ %}\n"
+    jinja: "{% set first_lang=\"\" %}\n{% if utterance_meta.lang == \"english\" %}\n\
+      {% if dialogue_history|length > 0 %}\nGiven the following dialogue between person\
+      \ A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
+      \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.ref }}{%\
+      \ endif %}\n{% endfor %} \nTranslate the next utterance in the dialogue into\
+      \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
+      \ %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{\
+      \ orig }}\n\n||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
+      \ %}: {{ ref }}\n{% endif %}\n{% endif %}\n"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -432,9 +434,9 @@ templates:
     answer_choices: yes || no
     id: dddd6f35-5a01-431b-b59b-e9a8e76377aa
     jinja: "{% set options = [\"word choice\", \"grammar\", \"style\", \"coherence\"\
-      , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% if utterance_meta.lang\
-      \ == \"french\" %}\n{% if options[label] in utterance_meta.eval_problems %}{%\
-      \ set reply=0 %}{% else %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
+      , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% set reply=0 %}\n\
+      {% if utterance_meta.lang == \"french\" %}\n{% if options[label] in utterance_meta.eval_problems\
+      \ %}{% set reply=0 %}{% else %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
       \ > 0 %}\n{% if dialogue_history|length > 0 %}\nGiven the following dialogue\
       \ between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\

From 44c7551015fdc2e55ed410c790ad312890599c2d Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Fri, 29 Apr 2022 02:02:48 +0200
Subject: [PATCH 05/21] declared more variables at beginning of templates

---
 .../templates/rbawden/DiaBLa/templates.yaml        | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index 6f5afac93..2b8c59970 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -54,8 +54,9 @@ templates:
     id: 123f36ee-c808-4cf0-aa5c-ed1f03ac3b69
     jinja: "{% set options = [\"word choice\", \"grammar\", \"style\", \"coherence\"\
       , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% set reply=0 %}\n\
-      {% if utterance_meta.lang == \"english\" %}\n{% if options[label] in utterance_meta.eval_problems\
-      \ %}{% set reply=0 %}{% else %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
+      {% set first_lang=\"\" %}\n{% if utterance_meta.lang == \"english\" %}\n{% if\
+      \ options[label] in utterance_meta.eval_problems %}{% set reply=0 %}{% else\
+      \ %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
       \ > 0 %}\n{% if dialogue_history|length > 0 %}\nGiven the following dialogue\
       \ between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
@@ -119,8 +120,8 @@ templates:
     id: 28ea04f4-338e-40cf-8730-4a794b5b64b2
     jinja: "{% set options = [\"word choice\", \"grammar\", \"style\", \"coherence\"\
       , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% set reply=0 %}\n\
-      {% if options[label] in utterance_meta.eval_problems %}{% set reply=0 %}{% else\
-      \ %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
+      {% set first_lang=\"\" %}\n{% if options[label] in utterance_meta.eval_problems\
+      \ %}{% set reply=0 %}{% else %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
       \ > 0 %}\n{% if dialogue_history|length > 0 %}\nGiven the following dialogue\
       \ between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
@@ -435,8 +436,9 @@ templates:
     id: dddd6f35-5a01-431b-b59b-e9a8e76377aa
     jinja: "{% set options = [\"word choice\", \"grammar\", \"style\", \"coherence\"\
       , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% set reply=0 %}\n\
-      {% if utterance_meta.lang == \"french\" %}\n{% if options[label] in utterance_meta.eval_problems\
-      \ %}{% set reply=0 %}{% else %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
+      {% set first_lang=\"\" %}\n{% if utterance_meta.lang == \"french\" %}\n{% if\
+      \ options[label] in utterance_meta.eval_problems %}{% set reply=0 %}{% else\
+      \ %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
       \ > 0 %}\n{% if dialogue_history|length > 0 %}\nGiven the following dialogue\
       \ between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\

From 9379706dd2d9c1a21966ce8d5820b8d0867cbbaf Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Fri, 29 Apr 2022 02:12:43 +0200
Subject: [PATCH 06/21] declared more variables at beginning of templates

---
 .../templates/rbawden/DiaBLa/templates.yaml   | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index 2b8c59970..32dff34be 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -77,17 +77,17 @@ templates:
   1798dee9-8a9c-45ed-87d8-237ec7dbf5ca: !Template
     answer_choices: null
     id: 1798dee9-8a9c-45ed-87d8-237ec7dbf5ca
-    jinja: "{% if utterance_meta.lang == \"english\" %}\n{% if dialogue_history|length\
-      \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
-      \ set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous\
-      \ in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang\
-      \ %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang\
-      \ %}{{ previous.orig }}{% else %}{{ previous.ref }}{% endif %}\n{% endfor %}\
-      \ \nTranslate the next utterance in the dialogue into {% if utterance_meta.lang\
-      \ == \"english\" %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif\
-      \ %}\n"
+    jinja: "{% set first_lang=\"\" %}\n{% if utterance_meta.lang == \"english\" %}\n\
+      {% if dialogue_history|length > 0 %}\nGiven the following dialogue between person\
+      \ A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
+      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
+      \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.ref }}{%\
+      \ endif %}\n{% endfor %} \nTranslate the next utterance in the dialogue into\
+      \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
+      \ %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{\
+      \ orig }}\n\n||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
+      \ %}: {{ ref }}\n{% endif %}\n{% endif %}\n"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:

From f5fc2a92a4041e8c6d88dd511ba48155a6226e7d Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Fri, 29 Apr 2022 02:21:11 +0200
Subject: [PATCH 07/21] corrected ref for mt in one template

---
 promptsource/templates/rbawden/DiaBLa/templates.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index 32dff34be..146e84adf 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -418,7 +418,7 @@ templates:
       \ A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
       \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
-      \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.ref }}{%\
+      \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
       \ endif %}\n{% endfor %} \nTranslate the next utterance in the dialogue into\
       \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
       \ %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{\

From ec3d5e853356c0aeaa6f3c294969db9b73d1ee65 Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Fri, 29 Apr 2022 02:22:32 +0200
Subject: [PATCH 08/21] corrected ref for mt in one template

---
 promptsource/templates/rbawden/DiaBLa/templates.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index 146e84adf..287c306f6 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -224,7 +224,7 @@ templates:
       \ A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
       \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
-      \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
+      \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.ref }}{%\
       \ endif %}\n{% endfor %} \nTranslate the next utterance in the dialogue into\
       \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
       \ %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{\

From ffee1479bd29bfb3fc052e07af1ee719d09768bf Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Fri, 29 Apr 2022 16:47:55 +0200
Subject: [PATCH 09/21] moved condition to just target side rather than around
 entire prompt

---
 .../templates/rbawden/DiaBLa/templates.yaml   | 310 +++++++++---------
 1 file changed, 152 insertions(+), 158 deletions(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index 287c306f6..46865c493 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -3,25 +3,13 @@ templates:
   0e55b858-92b0-4e5b-923b-0d23518035e3: !Template
     answer_choices: null
     id: 0e55b858-92b0-4e5b-923b-0d23518035e3
-    jinja: '{% set previous_ref = "" %}
-
-      {% if utterance_meta.lang == "french" %}
-
-      {% if dialogue_history|length > 0 %}
-
-
-      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
-      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
-      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
-      %}{% endif %}" translates into English as: {{ previous_ref }}
-
-
-      "{{ orig }}" translates into English as: ||| {{ ref }}
-
-
-      {% endif %}
-
-      {% endif %}'
+    jinja: "{% set previous_ref = \"\" %}\n{% if dialogue_history|length > 0 %}\n\"\
+      {% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{ dialogue_history[-1].orig\
+      \ }}{% set previous_ref = dialogue_history[-1].ref %}{% else %}{{ dialogue_history[-1].ref\
+      \ }}{% set previous_ref = dialogue_history[-1].orig %}{% endif %}\" translates\
+      \ into French as: {{ previous_ref }}{% endif %}\n\n\"{{ orig }}\" translates\
+      \ into English as: ||| \n{% if utterance_meta.lang == \"french\" %}\n{% if dialogue_history|length\
+      \ > 0 %}\n{{ ref }}\n{% endif %}\n{% endif %}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -32,16 +20,15 @@ templates:
   0e841022-eb3c-4f5a-b57a-0cbd3d628b68: !Template
     answer_choices: null
     id: 0e841022-eb3c-4f5a-b57a-0cbd3d628b68
-    jinja: "{% set first_lang=\"\" %}\n{% if utterance_meta.lang == \"english\" %}\n\
-      {% if dialogue_history|length > 0 %}\nGiven the following dialogue between person\
-      \ A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+    jinja: "{% set first_lang=\"\" %}\n{% if dialogue_history|length > 0 %}\nGiven\
+      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ previous.orig }}\n{% endfor %}\
-      \ \nTranslate the next utterance in the dialogue into {% if utterance_meta.lang\
-      \ == \"english\" %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif\
-      \ %}"
+      \ == first_lang %}A{% else %}B{% endif %}: {{ previous.orig }}\n{% endfor %}{%\
+      \ endif %}\nTranslate the next utterance in the dialogue into French:\n{% if\
+      \ utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n\
+      ||| \n{% if utterance_meta.lang == \"english\" %}\n{% if dialogue_history|length\
+      \ > 0 %}\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}:\
+      \ {{ ref }}\n{% endif %}\n{% endif %}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -52,20 +39,44 @@ templates:
   123f36ee-c808-4cf0-aa5c-ed1f03ac3b69: !Template
     answer_choices: yes || no
     id: 123f36ee-c808-4cf0-aa5c-ed1f03ac3b69
-    jinja: "{% set options = [\"word choice\", \"grammar\", \"style\", \"coherence\"\
-      , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% set reply=0 %}\n\
-      {% set first_lang=\"\" %}\n{% if utterance_meta.lang == \"english\" %}\n{% if\
-      \ options[label] in utterance_meta.eval_problems %}{% set reply=0 %}{% else\
-      \ %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
-      \ > 0 %}\n{% if dialogue_history|length > 0 %}\nGiven the following dialogue\
-      \ between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
-      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
-      \ != utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
-      \ endif %}{% endfor %} \n{% if utterance_meta.lang == first_lang %}A{% else\
-      \ %}B{% endif %}: {{ mt }}\n\nDoes the last utterance contain a {{ options[label]\
-      \ }} problem, {{ \"yes\" }} or {{ \"no\" }}?\n\n||| {{ [\"yes\", \"no\" ][reply]\
-      \ }}\n{% endif %}\n{% endif %}\n{% endif %}"
+    jinja: '{% set options = ["word choice", "grammar", "style", "coherence", "meaning"]
+      %}
+
+      {% set label = range(0,5)|random %}
+
+      {% set reply=0 %}
+
+      {% set first_lang="" %}
+
+
+      {% if options[label] in utterance_meta.eval_problems %}{% set reply=0 %}{% else
+      %}{% set reply=1 %}{% endif %}
+
+      {% if utterance_meta.eval_problems|length > 0 %}
+
+
+      Given the following dialogue between person A and person B:
+
+      {% if dialogue_history|length > 0 %}{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang
+      %}{% for previous in dialogue_history[-5:] %}
+
+      {% if previous.utterance_meta.lang == first_lang %}A{% else %}B{% endif %}:
+      {% if previous.utterance_meta.lang != utterance_meta.lang %}{{ previous.orig
+      }}{% else %}{{ previous.mt }}{% endif %}{% endfor %}{% endif %}
+
+      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ mt }}
+
+
+      Does the last utterance contain a {{ options[label] }} problem, {{ "yes" }}
+      or {{ "no" }}?
+
+
+      ||| {% if utterance_meta.eval_problems|length > 0 %}{% if utterance_meta.lang
+      == "english" %}{{ ["yes", "no" ][reply] }}{% endif %}
+
+      {% endif %}
+
+      {% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: true
       metrics:
@@ -77,17 +88,16 @@ templates:
   1798dee9-8a9c-45ed-87d8-237ec7dbf5ca: !Template
     answer_choices: null
     id: 1798dee9-8a9c-45ed-87d8-237ec7dbf5ca
-    jinja: "{% set first_lang=\"\" %}\n{% if utterance_meta.lang == \"english\" %}\n\
-      {% if dialogue_history|length > 0 %}\nGiven the following dialogue between person\
-      \ A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+    jinja: "{% set first_lang=\"\" %}\n{% if dialogue_history|length > 0 %}\nGiven\
+      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
       \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
       \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.ref }}{%\
-      \ endif %}\n{% endfor %} \nTranslate the next utterance in the dialogue into\
-      \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
-      \ %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{\
-      \ orig }}\n\n||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
-      \ %}: {{ ref }}\n{% endif %}\n{% endif %}\n"
+      \ endif %}\n{% endfor %}{% endif %}\nTranslate the next utterance in the dialogue\
+      \ into French:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
+      \ %}: {{ orig }}\n\n||| \n{% if utterance_meta.lang == \"english\" %}\n{% if\
+      \ dialogue_history|length > 0 %}\n{% if utterance_meta.lang == first_lang %}A{%\
+      \ else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif %}\n"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -141,22 +151,13 @@ templates:
   2abc728f-e7fc-44fc-ba2a-ad78ae2469a9: !Template
     answer_choices: null
     id: 2abc728f-e7fc-44fc-ba2a-ad78ae2469a9
-    jinja: '{% set previous_ref = "" %}{% if utterance_meta.lang == "english" %}
-
-      {% if dialogue_history|length > 0 %}
-
-      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
-      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
-      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
-      %}{% endif %}" translates as: {{ previous_ref }}
-
-
-      "{{ orig }}" translates as: ||| {{ ref }}
-
-
-      {% endif %}
-
-      {% endif %}'
+    jinja: "{% set previous_ref = \"\" %}\n{% if dialogue_history|length > 0 %}\n\"\
+      {% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{ dialogue_history[-1].orig\
+      \ }}{% set previous_ref = dialogue_history[-1].ref %}{% else %}{{ dialogue_history[-1].ref\
+      \ }}{% set previous_ref = dialogue_history[-1].orig %}{% endif %}\" translates\
+      \ as: {{ previous_ref }}{% endif %}\n\n\"{{ orig }}\" translates as: ||| \n\
+      {% if dialogue_history|length > 0 %}\n{% if utterance_meta.lang == \"english\"\
+      \ %}\n{{ ref }}\n{% endif %}\n{% endif %}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -167,22 +168,13 @@ templates:
   40c1ee21-3cf4-4efc-b902-d3aab0a79c40: !Template
     answer_choices: null
     id: 40c1ee21-3cf4-4efc-b902-d3aab0a79c40
-    jinja: '{% set previous_ref = "" %}{% if utterance_meta.lang == "french" %}
-
-      {% if dialogue_history|length > 0 %}
-
-      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
-      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
-      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
-      %}{% endif %}" translates as: {{ previous_ref }}
-
-
-      "{{ orig }}" translates as: ||| {{ ref }}
-
-
-      {% endif %}
-
-      {% endif %}'
+    jinja: "{% set previous_ref = \"\" %}\n{% if dialogue_history|length > 0 %}\n\"\
+      {% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{ dialogue_history[-1].orig\
+      \ }}{% set previous_ref = dialogue_history[-1].ref %}{% else %}{{ dialogue_history[-1].ref\
+      \ }}{% set previous_ref = dialogue_history[-1].orig %}{% endif %}\" translates\
+      \ as: {{ previous_ref }}{% endif %}\n\n\"{{ orig }}\" translates as: ||| \n\
+      {% if dialogue_history|length > 0 %}\n{% if utterance_meta.lang == \"french\"\
+      \ %}\n{{ ref }}\n{% endif %}\n{% endif %}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -219,17 +211,16 @@ templates:
   702f1f50-d6b0-4e65-a8e6-b449257f1e35: !Template
     answer_choices: null
     id: 702f1f50-d6b0-4e65-a8e6-b449257f1e35
-    jinja: "{% set first_lang=\"\" %}\n{% if utterance_meta.lang == \"french\" %}\n\
-      {% if dialogue_history|length > 0 %}\nGiven the following dialogue between person\
-      \ A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+    jinja: "{% set first_lang=\"\" %}\n{% if dialogue_history|length > 0 %}\nGiven\
+      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
       \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
       \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.ref }}{%\
-      \ endif %}\n{% endfor %} \nTranslate the next utterance in the dialogue into\
-      \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
-      \ %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{\
-      \ orig }}\n\n||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
-      \ %}: {{ ref }}\n{% endif %}\n{% endif %}\n"
+      \ endif %}\n{% endfor %}{% endif %}\nTranslate the next utterance in the dialogue\
+      \ into English:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
+      \ %}: {{ orig }}\n\n||| \n{% if utterance_meta.lang == \"french\" %}\n{% if\
+      \ dialogue_history|length > 0 %}\n{% if utterance_meta.lang == first_lang %}A{%\
+      \ else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif %}\n"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -240,17 +231,16 @@ templates:
   72759632-7764-4493-9a6e-8f1c8b337bde: !Template
     answer_choices: null
     id: 72759632-7764-4493-9a6e-8f1c8b337bde
-    jinja: "{% set first_lang=\"\" %}\n{% if utterance_meta.lang == \"french\" %}\n\
-      {% if dialogue_history|length > 0 %}\nGiven the following dialogue between person\
-      \ A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+    jinja: "{% set first_lang=\"\" %}\n\n{% if dialogue_history|length > 0 %}\nGiven\
+      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
       \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
       \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
-      \ endif %}\n{% endfor %} \nTranslate the next utterance in the dialogue into\
-      \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
-      \ %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{\
-      \ orig }}\n\n||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
-      \ %}: {{ ref }}\n{% endif %}\n{% endif %}\n"
+      \ endif %}\n{% endfor %}{% endif %}\nTranslate the next utterance in the dialogue\
+      \ into English:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
+      \ %}: {{ orig }}\n\n||| \n{% if utterance_meta.lang == \"french\" %}\n{% if\
+      \ dialogue_history|length > 0 %}\n{% if utterance_meta.lang == first_lang %}A{%\
+      \ else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif %}\n"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -261,11 +251,8 @@ templates:
   72c1cbbd-956f-4721-9cee-fd50148cf77d: !Template
     answer_choices: null
     id: 72c1cbbd-956f-4721-9cee-fd50148cf77d
-    jinja: '{% if utterance_meta.lang == "french"  %}
-
-      Translate the following into English: "{{ orig }}" ||| {{ ref }}
-
-      {% endif %}'
+    jinja: 'Translate the following into English: "{{ orig }}" ||| {% if utterance_meta.lang
+      == "french"  %}{{ ref }}{% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -276,16 +263,15 @@ templates:
   836c5096-2fd6-478c-89e8-39a050347d28: !Template
     answer_choices: null
     id: 836c5096-2fd6-478c-89e8-39a050347d28
-    jinja: "{% set first_lang=\"\" %}\n{% if utterance_meta.lang == \"french\" %}\n\
-      {% if dialogue_history|length > 0 %}\nGiven the following dialogue between person\
-      \ A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+    jinja: "{% set first_lang=\"\" %}\n\n{% if dialogue_history|length > 0 %}\nGiven\
+      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ previous.orig }}\n{% endfor %}\
-      \ \nTranslate the next utterance in the dialogue into {% if utterance_meta.lang\
-      \ == \"english\" %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif\
-      \ %}"
+      \ == first_lang %}A{% else %}B{% endif %}: {{ previous.orig }}\n{% endfor %}{%\
+      \ endif %}\nTranslate the next utterance in the dialogue into English:\n{% if\
+      \ utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n\
+      ||| \n\n{% if utterance_meta.lang == \"french\" %}\n{% if dialogue_history|length\
+      \ > 0 %}\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}:\
+      \ {{ ref }}\n{% endif %}\n{% endif %}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -326,11 +312,8 @@ templates:
   96da9c0e-e606-4283-8494-7129c75b4d02: !Template
     answer_choices: null
     id: 96da9c0e-e606-4283-8494-7129c75b4d02
-    jinja: '{% if utterance_meta.lang == "english"  %}
-
-      Translate the following into French: "{{ orig }}" ||| {{ ref }}
-
-      {% endif %}'
+    jinja: 'Translate the following into French: "{{ orig }}" ||| {% if utterance_meta.lang
+      == "english"  %}{{ ref }}{% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -341,25 +324,13 @@ templates:
   97d56019-a7df-4733-aed0-332c222e9079: !Template
     answer_choices: null
     id: 97d56019-a7df-4733-aed0-332c222e9079
-    jinja: '{% set previous_ref = "" %}
-
-      {% if utterance_meta.lang == "english" %}
-
-      {% if dialogue_history|length > 0 %}
-
-
-      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
-      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
-      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
-      %}{% endif %}" translates into French as: {{ previous_ref }}
-
-
-      "{{ orig }}" translates into French as: ||| {{ ref }}
-
-
-      {% endif %}
-
-      {% endif %}'
+    jinja: "{% set previous_ref = \"\" %}\n{% if dialogue_history|length > 0 %}\n\"\
+      {% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{ dialogue_history[-1].orig\
+      \ }}{% set previous_ref = dialogue_history[-1].ref %}{% else %}{{ dialogue_history[-1].ref\
+      \ }}{% set previous_ref = dialogue_history[-1].orig %}{% endif %}\" translates\
+      \ into French as: {{ previous_ref }}{% endif %}\n\n\"{{ orig }}\" translates\
+      \ into French as: ||| \n{% if utterance_meta.lang == \"english\" %}\n{% if dialogue_history|length\
+      \ > 0 %}\n{{ ref }}\n{% endif %}\n{% endif %}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -413,17 +384,16 @@ templates:
   d068ba77-ce75-4de3-a170-ca0a7cd3f8d1: !Template
     answer_choices: null
     id: d068ba77-ce75-4de3-a170-ca0a7cd3f8d1
-    jinja: "{% set first_lang=\"\" %}\n{% if utterance_meta.lang == \"english\" %}\n\
-      {% if dialogue_history|length > 0 %}\nGiven the following dialogue between person\
-      \ A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
+    jinja: "{% set first_lang=\"\" %}\n{% if dialogue_history|length > 0 %}\nGiven\
+      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
       \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
       \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
-      \ endif %}\n{% endfor %} \nTranslate the next utterance in the dialogue into\
-      \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
-      \ %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{\
-      \ orig }}\n\n||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
-      \ %}: {{ ref }}\n{% endif %}\n{% endif %}\n"
+      \ endif %}\n{% endfor %}{% endif %}\nTranslate the next utterance in the dialogue\
+      \ into French:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
+      \ %}: {{ orig }}\n\n||| \n{% if utterance_meta.lang == \"english\" %}\n{% if\
+      \ dialogue_history|length > 0 %}\n{% if utterance_meta.lang == first_lang %}A{%\
+      \ else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif %}\n"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -434,20 +404,44 @@ templates:
   dddd6f35-5a01-431b-b59b-e9a8e76377aa: !Template
     answer_choices: yes || no
     id: dddd6f35-5a01-431b-b59b-e9a8e76377aa
-    jinja: "{% set options = [\"word choice\", \"grammar\", \"style\", \"coherence\"\
-      , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% set reply=0 %}\n\
-      {% set first_lang=\"\" %}\n{% if utterance_meta.lang == \"french\" %}\n{% if\
-      \ options[label] in utterance_meta.eval_problems %}{% set reply=0 %}{% else\
-      \ %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
-      \ > 0 %}\n{% if dialogue_history|length > 0 %}\nGiven the following dialogue\
-      \ between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
-      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
-      \ != utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
-      \ endif %}{% endfor %} \n{% if utterance_meta.lang == first_lang %}A{% else\
-      \ %}B{% endif %}: {{ mt }}\n\nDoes the last utterance contain a {{ options[label]\
-      \ }} problem, {{ \"yes\" }} or {{ \"no\" }}?\n\n||| {{ [\"yes\", \"no\" ][reply]\
-      \ }}\n{% endif %}\n{% endif %}\n{% endif %}"
+    jinja: '{% set options = ["word choice", "grammar", "style", "coherence", "meaning"]
+      %}
+
+      {% set label = range(0,5)|random %}
+
+      {% set reply=0 %}
+
+      {% set first_lang="" %}
+
+
+      {% if options[label] in utterance_meta.eval_problems %}{% set reply=0 %}{% else
+      %}{% set reply=1 %}{% endif %}
+
+      {% if utterance_meta.eval_problems|length > 0 %}
+
+
+      Given the following dialogue between person A and person B:
+
+      {% if dialogue_history|length > 0 %}{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang
+      %}{% for previous in dialogue_history[-5:] %}
+
+      {% if previous.utterance_meta.lang == first_lang %}A{% else %}B{% endif %}:
+      {% if previous.utterance_meta.lang != utterance_meta.lang %}{{ previous.orig
+      }}{% else %}{{ previous.mt }}{% endif %}{% endfor %}{% endif %}
+
+      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ mt }}
+
+
+      Does the last utterance contain a {{ options[label] }} problem, {{ "yes" }}
+      or {{ "no" }}?
+
+
+      ||| {% if utterance_meta.eval_problems|length > 0 %}{% if utterance_meta.lang
+      == "french" %}{{ ["yes", "no" ][reply] }}{% endif %}
+
+      {% endif %}
+
+      {% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: true
       metrics:

From 7659b3d31992450c82e1af78f87fe19745869b28 Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Fri, 29 Apr 2022 17:20:51 +0200
Subject: [PATCH 10/21] allow template even when no context (beginning of
 dialogue)

---
 .../templates/rbawden/DiaBLa/templates.yaml   | 349 ++++++++++++------
 1 file changed, 235 insertions(+), 114 deletions(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index 46865c493..e2243d726 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -3,13 +3,18 @@ templates:
   0e55b858-92b0-4e5b-923b-0d23518035e3: !Template
     answer_choices: null
     id: 0e55b858-92b0-4e5b-923b-0d23518035e3
-    jinja: "{% set previous_ref = \"\" %}\n{% if dialogue_history|length > 0 %}\n\"\
-      {% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{ dialogue_history[-1].orig\
-      \ }}{% set previous_ref = dialogue_history[-1].ref %}{% else %}{{ dialogue_history[-1].ref\
-      \ }}{% set previous_ref = dialogue_history[-1].orig %}{% endif %}\" translates\
-      \ into French as: {{ previous_ref }}{% endif %}\n\n\"{{ orig }}\" translates\
-      \ into English as: ||| \n{% if utterance_meta.lang == \"french\" %}\n{% if dialogue_history|length\
-      \ > 0 %}\n{{ ref }}\n{% endif %}\n{% endif %}"
+    jinja: '{% set previous_ref = "" %}
+
+      {% if dialogue_history|length > 0 %}
+
+      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
+      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
+      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
+      %}{% endif %}" translates into French as: {{ previous_ref }}{% endif %}
+
+
+      "{{ orig }}" translates into English as: ||| {% if utterance_meta.lang == "french"
+      %}{{ ref }}{% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -20,15 +25,26 @@ templates:
   0e841022-eb3c-4f5a-b57a-0cbd3d628b68: !Template
     answer_choices: null
     id: 0e841022-eb3c-4f5a-b57a-0cbd3d628b68
-    jinja: "{% set first_lang=\"\" %}\n{% if dialogue_history|length > 0 %}\nGiven\
-      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
-      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ previous.orig }}\n{% endfor %}{%\
-      \ endif %}\nTranslate the next utterance in the dialogue into French:\n{% if\
-      \ utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n\
-      ||| \n{% if utterance_meta.lang == \"english\" %}\n{% if dialogue_history|length\
-      \ > 0 %}\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}:\
-      \ {{ ref }}\n{% endif %}\n{% endif %}"
+    jinja: '{% set first_lang="" %}
+
+      {% if dialogue_history|length > 0 %}
+
+      Given the following dialogue between person A and person B:
+
+
+      {% set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous
+      in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang
+      %}A{% else %}B{% endif %}: {{ previous.orig }}
+
+      {% endfor %}{% endif %}
+
+      Translate the next utterance in the dialogue into French:
+
+      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}
+
+
+      ||| {% if utterance_meta.lang == "english" %}{% if utterance_meta.lang == first_lang
+      %}A{% else %}B{% endif %}: {{ ref }}{% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -88,16 +104,29 @@ templates:
   1798dee9-8a9c-45ed-87d8-237ec7dbf5ca: !Template
     answer_choices: null
     id: 1798dee9-8a9c-45ed-87d8-237ec7dbf5ca
-    jinja: "{% set first_lang=\"\" %}\n{% if dialogue_history|length > 0 %}\nGiven\
-      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
-      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
-      \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.ref }}{%\
-      \ endif %}\n{% endfor %}{% endif %}\nTranslate the next utterance in the dialogue\
-      \ into French:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
-      \ %}: {{ orig }}\n\n||| \n{% if utterance_meta.lang == \"english\" %}\n{% if\
-      \ dialogue_history|length > 0 %}\n{% if utterance_meta.lang == first_lang %}A{%\
-      \ else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif %}\n"
+    jinja: '{% set first_lang="" %}
+
+      {% if dialogue_history|length > 0 %}
+
+      Given the following dialogue between person A and person B:
+
+
+      {% set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous
+      in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang
+      %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang
+      %}{{ previous.orig }}{% else %}{{ previous.ref }}{% endif %}
+
+      {% endfor %}{% endif %}
+
+      Translate the next utterance in the dialogue into French:
+
+      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}
+
+
+      ||| {% if utterance_meta.lang == "english" %}{% if utterance_meta.lang == first_lang
+      %}A{% else %}B{% endif %}: {{ ref }}{% endif %}
+
+      '
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -117,7 +146,7 @@ templates:
       \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
       \ %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{\
       \ orig }}\n\n||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
-      \ %}: {{ ref }}\n{% endif %}"
+      \ %}: {{ ref }}{% endif %}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -151,13 +180,18 @@ templates:
   2abc728f-e7fc-44fc-ba2a-ad78ae2469a9: !Template
     answer_choices: null
     id: 2abc728f-e7fc-44fc-ba2a-ad78ae2469a9
-    jinja: "{% set previous_ref = \"\" %}\n{% if dialogue_history|length > 0 %}\n\"\
-      {% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{ dialogue_history[-1].orig\
-      \ }}{% set previous_ref = dialogue_history[-1].ref %}{% else %}{{ dialogue_history[-1].ref\
-      \ }}{% set previous_ref = dialogue_history[-1].orig %}{% endif %}\" translates\
-      \ as: {{ previous_ref }}{% endif %}\n\n\"{{ orig }}\" translates as: ||| \n\
-      {% if dialogue_history|length > 0 %}\n{% if utterance_meta.lang == \"english\"\
-      \ %}\n{{ ref }}\n{% endif %}\n{% endif %}"
+    jinja: '{% set previous_ref = "" %}
+
+      {% if dialogue_history|length > 0 %}
+
+      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
+      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
+      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
+      %}{% endif %}" translates as: {{ previous_ref }}{% endif %}
+
+
+      "{{ orig }}" translates as: ||| {% if utterance_meta.lang == "english" %}{{
+      ref }}{% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -168,13 +202,18 @@ templates:
   40c1ee21-3cf4-4efc-b902-d3aab0a79c40: !Template
     answer_choices: null
     id: 40c1ee21-3cf4-4efc-b902-d3aab0a79c40
-    jinja: "{% set previous_ref = \"\" %}\n{% if dialogue_history|length > 0 %}\n\"\
-      {% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{ dialogue_history[-1].orig\
-      \ }}{% set previous_ref = dialogue_history[-1].ref %}{% else %}{{ dialogue_history[-1].ref\
-      \ }}{% set previous_ref = dialogue_history[-1].orig %}{% endif %}\" translates\
-      \ as: {{ previous_ref }}{% endif %}\n\n\"{{ orig }}\" translates as: ||| \n\
-      {% if dialogue_history|length > 0 %}\n{% if utterance_meta.lang == \"french\"\
-      \ %}\n{{ ref }}\n{% endif %}\n{% endif %}"
+    jinja: '{% set previous_ref = "" %}
+
+      {% if dialogue_history|length > 0 %}
+
+      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
+      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
+      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
+      %}{% endif %}" translates as: {{ previous_ref }}{% endif %}
+
+
+      "{{ orig }}" translates as: ||| {% if utterance_meta.lang == "french" %}{{ ref
+      }}{% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -187,20 +226,21 @@ templates:
     id: 6a01fbe6-d5ec-4ad9-a2ee-3c48ed095885
     jinja: '{% set previous_ref = "" %}{% set other_lang = "" %}
 
-      {% if dialogue_history|length > 0 %}{% if utterance_meta.lang == "french" %}{%
-      set other_lang = "English" %}{% else %}{% set other_lang = "French" %}{% endif
-      %}
+      {% if utterance_meta.lang == "french" %}{% set other_lang = "English" %}{% else
+      %}{% set other_lang = "French" %}
+
+      {% if dialogue_history|length > 0 %}{% endif %}
 
       "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
       dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
       else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
-      %}{% endif %}" translates into {{ other_lang }} as: {{ previous_ref }}
+      %}{% endif %}" translates into {{ other_lang }} as: {{ previous_ref }}{% endif
+      %}
 
 
       "{{ orig }}" translates into {{ other_lang }} as: ||| {{ ref }}
 
-
-      {% endif %}'
+      '
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -211,16 +251,29 @@ templates:
   702f1f50-d6b0-4e65-a8e6-b449257f1e35: !Template
     answer_choices: null
     id: 702f1f50-d6b0-4e65-a8e6-b449257f1e35
-    jinja: "{% set first_lang=\"\" %}\n{% if dialogue_history|length > 0 %}\nGiven\
-      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
-      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
-      \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.ref }}{%\
-      \ endif %}\n{% endfor %}{% endif %}\nTranslate the next utterance in the dialogue\
-      \ into English:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
-      \ %}: {{ orig }}\n\n||| \n{% if utterance_meta.lang == \"french\" %}\n{% if\
-      \ dialogue_history|length > 0 %}\n{% if utterance_meta.lang == first_lang %}A{%\
-      \ else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif %}\n"
+    jinja: '{% set first_lang="" %}
+
+      {% if dialogue_history|length > 0 %}
+
+      Given the following dialogue between person A and person B:
+
+
+      {% set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous
+      in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang
+      %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang
+      %}{{ previous.orig }}{% else %}{{ previous.ref }}{% endif %}
+
+      {% endfor %}{% endif %}
+
+      Translate the next utterance in the dialogue into English:
+
+      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}
+
+
+      ||| {% if utterance_meta.lang == "french" %}{% if utterance_meta.lang == first_lang
+      %}A{% else %}B{% endif %}: {{ ref }}{% endif %}
+
+      '
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -231,16 +284,30 @@ templates:
   72759632-7764-4493-9a6e-8f1c8b337bde: !Template
     answer_choices: null
     id: 72759632-7764-4493-9a6e-8f1c8b337bde
-    jinja: "{% set first_lang=\"\" %}\n\n{% if dialogue_history|length > 0 %}\nGiven\
-      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
-      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
-      \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
-      \ endif %}\n{% endfor %}{% endif %}\nTranslate the next utterance in the dialogue\
-      \ into English:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
-      \ %}: {{ orig }}\n\n||| \n{% if utterance_meta.lang == \"french\" %}\n{% if\
-      \ dialogue_history|length > 0 %}\n{% if utterance_meta.lang == first_lang %}A{%\
-      \ else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif %}\n"
+    jinja: '{% set first_lang="" %}
+
+
+      {% if dialogue_history|length > 0 %}
+
+      Given the following dialogue between person A and person B:
+
+
+      {% set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous
+      in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang
+      %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang
+      %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
+
+      {% endfor %}{% endif %}
+
+      Translate the next utterance in the dialogue into English:
+
+      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}
+
+
+      ||| {% if utterance_meta.lang == "french" %}{% if utterance_meta.lang == first_lang
+      %}A{% else %}B{% endif %}: {{ ref }}{% endif %}
+
+      '
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -263,15 +330,27 @@ templates:
   836c5096-2fd6-478c-89e8-39a050347d28: !Template
     answer_choices: null
     id: 836c5096-2fd6-478c-89e8-39a050347d28
-    jinja: "{% set first_lang=\"\" %}\n\n{% if dialogue_history|length > 0 %}\nGiven\
-      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
-      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ previous.orig }}\n{% endfor %}{%\
-      \ endif %}\nTranslate the next utterance in the dialogue into English:\n{% if\
-      \ utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n\
-      ||| \n\n{% if utterance_meta.lang == \"french\" %}\n{% if dialogue_history|length\
-      \ > 0 %}\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}:\
-      \ {{ ref }}\n{% endif %}\n{% endif %}"
+    jinja: '{% set first_lang="" %}
+
+
+      {% if dialogue_history|length > 0 %}
+
+      Given the following dialogue between person A and person B:
+
+
+      {% set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous
+      in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang
+      %}A{% else %}B{% endif %}: {{ previous.orig }}
+
+      {% endfor %}{% endif %}
+
+      Translate the next utterance in the dialogue into English:
+
+      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}
+
+
+      ||| {% if utterance_meta.lang == "french" %}{% if utterance_meta.lang == first_lang
+      %}A{% else %}B{% endif %}: {{ ref }}{% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -294,14 +373,27 @@ templates:
   93f5256d-bd93-4056-b466-152b55860d02: !Template
     answer_choices: null
     id: 93f5256d-bd93-4056-b466-152b55860d02
-    jinja: "{% set first_lang=\"\" %}\n{% if dialogue_history|length > 0 %}\nGiven\
-      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
-      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ previous.orig }}\n{% endfor %}\
-      \ \nTranslate the next utterance in the dialogue into {% if utterance_meta.lang\
-      \ == \"english\" %}French{% else %}English{% endif %}:\n{% if utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ orig }}\n\n||| {% if utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {{ ref }}\n{% endif %}"
+    jinja: '{% set first_lang="" %}
+
+      {% if dialogue_history|length > 0 %}
+
+      Given the following dialogue between person A and person B:
+
+
+      {% set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous
+      in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang
+      %}A{% else %}B{% endif %}: {{ previous.orig }}
+
+      {% endfor %}{% endif %}
+
+      Translate the next utterance in the dialogue into {% if utterance_meta.lang
+      == "english" %}French{% else %}English{% endif %}:
+
+      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}
+
+
+      ||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ ref
+      }}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -324,13 +416,18 @@ templates:
   97d56019-a7df-4733-aed0-332c222e9079: !Template
     answer_choices: null
     id: 97d56019-a7df-4733-aed0-332c222e9079
-    jinja: "{% set previous_ref = \"\" %}\n{% if dialogue_history|length > 0 %}\n\"\
-      {% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{ dialogue_history[-1].orig\
-      \ }}{% set previous_ref = dialogue_history[-1].ref %}{% else %}{{ dialogue_history[-1].ref\
-      \ }}{% set previous_ref = dialogue_history[-1].orig %}{% endif %}\" translates\
-      \ into French as: {{ previous_ref }}{% endif %}\n\n\"{{ orig }}\" translates\
-      \ into French as: ||| \n{% if utterance_meta.lang == \"english\" %}\n{% if dialogue_history|length\
-      \ > 0 %}\n{{ ref }}\n{% endif %}\n{% endif %}"
+    jinja: '{% set previous_ref = "" %}
+
+      {% if dialogue_history|length > 0 %}
+
+      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
+      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
+      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
+      %}{% endif %}" translates into French as: {{ previous_ref }}{% endif %}
+
+
+      "{{ orig }}" translates into French as: ||| {% if utterance_meta.lang == "english"
+      %}{{ ref }}{% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -364,16 +461,28 @@ templates:
   b61c81ec-29eb-47f8-a1c6-561264ac04f3: !Template
     answer_choices: null
     id: b61c81ec-29eb-47f8-a1c6-561264ac04f3
-    jinja: "{% set first_lang=\"\" %}\n{% if dialogue_history|length > 0 %}\nGiven\
-      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
-      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
-      \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
-      \ endif %}\n{% endfor %} \nTranslate the next utterance in the dialogue into\
-      \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
-      \ %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{\
-      \ orig }}\n\n||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
-      \ %}: {{ ref }}\n{% endif %}"
+    jinja: '{% set first_lang="" %}
+
+      {% if dialogue_history|length > 0 %}
+
+      Given the following dialogue between person A and person B:
+
+
+      {% set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous
+      in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang
+      %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang
+      %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
+
+      {% endfor %}{% endif %}
+
+      Translate the next utterance in the dialogue into {% if utterance_meta.lang
+      == "english" %}French{% else %}English{% endif %}:
+
+      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}
+
+
+      ||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ ref
+      }}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -384,16 +493,29 @@ templates:
   d068ba77-ce75-4de3-a170-ca0a7cd3f8d1: !Template
     answer_choices: null
     id: d068ba77-ce75-4de3-a170-ca0a7cd3f8d1
-    jinja: "{% set first_lang=\"\" %}\n{% if dialogue_history|length > 0 %}\nGiven\
-      \ the following dialogue between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
-      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
-      \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
-      \ endif %}\n{% endfor %}{% endif %}\nTranslate the next utterance in the dialogue\
-      \ into French:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
-      \ %}: {{ orig }}\n\n||| \n{% if utterance_meta.lang == \"english\" %}\n{% if\
-      \ dialogue_history|length > 0 %}\n{% if utterance_meta.lang == first_lang %}A{%\
-      \ else %}B{% endif %}: {{ ref }}\n{% endif %}\n{% endif %}\n"
+    jinja: '{% set first_lang="" %}
+
+      {% if dialogue_history|length > 0 %}
+
+      Given the following dialogue between person A and person B:
+
+
+      {% set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous
+      in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang
+      %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang
+      %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
+
+      {% endfor %}{% endif %}
+
+      Translate the next utterance in the dialogue into French:
+
+      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}
+
+
+      ||| {% if utterance_meta.lang == "english" %}{% if utterance_meta.lang == first_lang
+      %}A{% else %}B{% endif %}: {{ ref }}{% endif %}
+
+      '
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -463,13 +585,12 @@ templates:
       "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
       dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
       else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
-      %}{% endif %}" translates as: {{ previous_ref }}
+      %}{% endif %}" translates as: {{ previous_ref }}{% endif %}
 
 
       "{{ orig }}" translates as: ||| {{ ref }}
 
-
-      {% endif %}'
+      '
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:

From 88127d4f59edf17d7fdba9f9bf1c0140ca126687 Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Fri, 29 Apr 2022 18:22:12 +0200
Subject: [PATCH 11/21] corrected templates that use past history

---
 .../templates/rbawden/DiaBLa/templates.yaml   | 63 ++++++++-----------
 1 file changed, 25 insertions(+), 38 deletions(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index e2243d726..c19d3db1f 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -68,8 +68,6 @@ templates:
       {% if options[label] in utterance_meta.eval_problems %}{% set reply=0 %}{% else
       %}{% set reply=1 %}{% endif %}
 
-      {% if utterance_meta.eval_problems|length > 0 %}
-
 
       Given the following dialogue between person A and person B:
 
@@ -88,11 +86,7 @@ templates:
 
 
       ||| {% if utterance_meta.eval_problems|length > 0 %}{% if utterance_meta.lang
-      == "english" %}{{ ["yes", "no" ][reply] }}{% endif %}
-
-      {% endif %}
-
-      {% endif %}'
+      == "english" %}{{ ["yes", "no" ][reply] }}{% endif %}{% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: true
       metrics:
@@ -142,11 +136,11 @@ templates:
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
       \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
       \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.ref }}{%\
-      \ endif %}\n{% endfor %} \nTranslate the next utterance in the dialogue into\
-      \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
-      \ %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{\
-      \ orig }}\n\n||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
-      \ %}: {{ ref }}{% endif %}"
+      \ endif %}\n{% endfor %}{% endif %} \nTranslate the next utterance in the dialogue\
+      \ into {% if utterance_meta.lang == \"english\" %}French{% else %}English{%\
+      \ endif %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
+      \ %}: {{ orig }}\n\n||| {% if utterance_meta.lang == first_lang %}A{% else %}B{%\
+      \ endif %}: {{ ref }}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -160,16 +154,16 @@ templates:
     jinja: "{% set options = [\"word choice\", \"grammar\", \"style\", \"coherence\"\
       , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% set reply=0 %}\n\
       {% set first_lang=\"\" %}\n{% if options[label] in utterance_meta.eval_problems\
-      \ %}{% set reply=0 %}{% else %}{% set reply=1 %}{% endif %}\n{% if utterance_meta.eval_problems|length\
-      \ > 0 %}\n{% if dialogue_history|length > 0 %}\nGiven the following dialogue\
-      \ between person A and person B:\n\n{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang\
-      \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
-      \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
-      \ != utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{%\
-      \ endif %}{% endfor %} \n{% if utterance_meta.lang == first_lang %}A{% else\
-      \ %}B{% endif %}: {{ mt }}\n\nDoes the last utterance contain a {{ options[label]\
-      \ }} problem, {{ \"yes\" }} or {{ \"no\" }}?\n\n||| {{ [\"yes\", \"no\" ][reply]\
-      \ }}\n{% endif %}\n{% endif %}"
+      \ %}{% set reply=0 %}{% else %}{% set reply=1 %}{% endif %}\n{% if dialogue_history|length\
+      \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
+      \ set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous\
+      \ in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang\
+      \ %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang != utterance_meta.lang\
+      \ %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}{% endfor %}{%\
+      \ endif %} \n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
+      \ %}: {{ mt }}\n\nDoes the last utterance contain a {{ options[label] }} problem,\
+      \ {{ \"yes\" }} or {{ \"no\" }}?\n\n||| {% if utterance_meta.eval_problems|length\
+      \ > 0 %}{{ [\"yes\", \"no\" ][reply] }}{% endif %}"
     metadata: !TemplateMetadata
       choices_in_prompt: true
       metrics:
@@ -190,8 +184,8 @@ templates:
       %}{% endif %}" translates as: {{ previous_ref }}{% endif %}
 
 
-      "{{ orig }}" translates as: ||| {% if utterance_meta.lang == "english" %}{{
-      ref }}{% endif %}'
+      "{{ orig }}" translates as: ||| {% if dialogue_history|length > 0 %}{% if utterance_meta.lang
+      == "english" %}{{ ref }}{% endif %}{% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -227,9 +221,9 @@ templates:
     jinja: '{% set previous_ref = "" %}{% set other_lang = "" %}
 
       {% if utterance_meta.lang == "french" %}{% set other_lang = "English" %}{% else
-      %}{% set other_lang = "French" %}
+      %}{% set other_lang = "French" %}{% endif %}
 
-      {% if dialogue_history|length > 0 %}{% endif %}
+      {% if dialogue_history|length > 0 %}
 
       "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
       dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
@@ -440,17 +434,13 @@ templates:
     id: ac4c63da-32d2-40ac-aa7a-632e8ba42b4a
     jinja: '{% set label = range(0,2)|random %}
 
-      {% if ref != mt %}
-
       Which of the following translations of "{{ orig }}" is produced automatically?
 
       {{ "A" }}) {% if label == 0 %}{{ ref }}{% else %}{{ mt }}{% endif %}
 
       {{ "B" }}) {% if label == 0 %}{{ mt }}{% else %}{{ ref }}{% endif %}
 
-      ||| {{ ["A", "B"][label] }}
-
-      {% endif %}'
+      ||| {% if ref != mt %}{{ ["A", "B"][label] }}{% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: true
       metrics:
@@ -539,8 +529,6 @@ templates:
       {% if options[label] in utterance_meta.eval_problems %}{% set reply=0 %}{% else
       %}{% set reply=1 %}{% endif %}
 
-      {% if utterance_meta.eval_problems|length > 0 %}
-
 
       Given the following dialogue between person A and person B:
 
@@ -559,11 +547,9 @@ templates:
 
 
       ||| {% if utterance_meta.eval_problems|length > 0 %}{% if utterance_meta.lang
-      == "french" %}{{ ["yes", "no" ][reply] }}{% endif %}
-
-      {% endif %}
+      == "french" %}{{ ["yes", "no" ][reply] }}{% endif %}{% endif %}
 
-      {% endif %}'
+      '
     metadata: !TemplateMetadata
       choices_in_prompt: true
       metrics:
@@ -588,7 +574,8 @@ templates:
       %}{% endif %}" translates as: {{ previous_ref }}{% endif %}
 
 
-      "{{ orig }}" translates as: ||| {{ ref }}
+      "{{ orig }}" translates as: ||| {% if dialogue_history|length > 0 %}{{ ref }}{%
+      endif %}
 
       '
     metadata: !TemplateMetadata

From bf51583ef59855cc7b706fb740c2b62dd72f8ab1 Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Tue, 10 May 2022 00:05:08 +0200
Subject: [PATCH 12/21] corrected multiple choice answer field

---
 promptsource/templates/rbawden/DiaBLa/templates.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index c19d3db1f..e6a2230ed 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -559,7 +559,7 @@ templates:
     reference: Identify presence of notable errors, same language context (5), English
       translation
   eea8f47e-9bf5-4423-980b-58a9635c1f49: !Template
-    answer_choices: 1 previous context (same language) used for analogy, both directions
+    answer_choices: null
     id: eea8f47e-9bf5-4423-980b-58a9635c1f49
     jinja: '{% set previous_ref = "" %}{% set other_lang = "" %}
 

From 5aa91909dd2d70619fccd99f2b60e50cf6d78a1b Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Wed, 11 May 2022 16:57:59 +0200
Subject: [PATCH 13/21] updated templates - simplified some targets and added
 translation completion tasks (simpler?)

---
 .../templates/rbawden/DiaBLa/templates.yaml   | 260 ++++++++++++++----
 1 file changed, 202 insertions(+), 58 deletions(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index e6a2230ed..19243b92f 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -38,13 +38,11 @@ templates:
 
       {% endfor %}{% endif %}
 
-      Translate the next utterance in the dialogue into French:
+      Translate {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}''s
+      next utterance into French: {{ orig }}
 
-      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}
 
-
-      ||| {% if utterance_meta.lang == "english" %}{% if utterance_meta.lang == first_lang
-      %}A{% else %}B{% endif %}: {{ ref }}{% endif %}'
+      ||| {% if utterance_meta.lang == "english" %}{{ ref }}{% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -85,8 +83,8 @@ templates:
       or {{ "no" }}?
 
 
-      ||| {% if utterance_meta.eval_problems|length > 0 %}{% if utterance_meta.lang
-      == "english" %}{{ ["yes", "no" ][reply] }}{% endif %}{% endif %}'
+      ||| {% if utterance_meta.lang == "english" %}{{ ["yes", "no" ][reply] }}{% endif
+      %}'
     metadata: !TemplateMetadata
       choices_in_prompt: true
       metrics:
@@ -112,13 +110,11 @@ templates:
 
       {% endfor %}{% endif %}
 
-      Translate the next utterance in the dialogue into French:
-
-      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}
+      Translate {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}''s
+      next utterance into French: {{ orig }}
 
 
-      ||| {% if utterance_meta.lang == "english" %}{% if utterance_meta.lang == first_lang
-      %}A{% else %}B{% endif %}: {{ ref }}{% endif %}
+      ||| {% if utterance_meta.lang == "english" %} {{ ref }}{% endif %}
 
       '
     metadata: !TemplateMetadata
@@ -136,11 +132,10 @@ templates:
       \ %}{% for previous in dialogue_history[-5:] %}{% if previous.utterance_meta.lang\
       \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
       \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.ref }}{%\
-      \ endif %}\n{% endfor %}{% endif %} \nTranslate the next utterance in the dialogue\
-      \ into {% if utterance_meta.lang == \"english\" %}French{% else %}English{%\
-      \ endif %}:\n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
-      \ %}: {{ orig }}\n\n||| {% if utterance_meta.lang == first_lang %}A{% else %}B{%\
-      \ endif %}: {{ ref }}"
+      \ endif %}\n{% endfor %}{% endif %} \nTranslate {% if utterance_meta.lang ==\
+      \ first_lang %}A{% else %}B{% endif %}'s next utterance in the dialogue into\
+      \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
+      \ %}: {{ orig }}\n\n||| {{ ref }}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -157,13 +152,12 @@ templates:
       \ %}{% set reply=0 %}{% else %}{% set reply=1 %}{% endif %}\n{% if dialogue_history|length\
       \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
       \ set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous\
-      \ in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang\
+      \ in dialogue_history[-5:] %}\n{% if previous.utterance_meta.lang == first_lang\
       \ %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang != utterance_meta.lang\
       \ %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}{% endfor %}{%\
       \ endif %} \n{% if utterance_meta.lang == first_lang %}A{% else %}B{% endif\
       \ %}: {{ mt }}\n\nDoes the last utterance contain a {{ options[label] }} problem,\
-      \ {{ \"yes\" }} or {{ \"no\" }}?\n\n||| {% if utterance_meta.eval_problems|length\
-      \ > 0 %}{{ [\"yes\", \"no\" ][reply] }}{% endif %}"
+      \ {{ \"yes\" }} or {{ \"no\" }}?\n\n||| {{ [\"yes\", \"no\" ][reply] }}"
     metadata: !TemplateMetadata
       choices_in_prompt: true
       metrics:
@@ -193,6 +187,60 @@ templates:
       original_task: true
     name: MT_1-context-analogy_infer-lang_en2fr
     reference: 1 previous context (same language) used for analogy, English-to-French
+  35822dda-6f09-418a-a2a3-2178d94203de: !Template
+    answer_choices: null
+    id: 35822dda-6f09-418a-a2a3-2178d94203de
+    jinja: '{% set target_lang = "english" %}{% if utterance_meta.lang == "english"
+      %}{% set target_lang = "french" %}{% endif %}
+
+      {% for previous in dialogue_history[-2:] %}
+
+      {{ previous.orig }}{% endfor %}
+
+      {{ orig }}
+
+
+      The {% if utterance_meta.lang == "english" %}French{% else %}English{% endif
+      %} translation is:
+
+      {% for previous in dialogue_history[-2:] %}{% if previous.utterance_meta.lang
+      == target_lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
+
+      {% endfor %} ||| {% if utterance_meta.lang =="french" %}{{ ref }}{% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT-2-context-complete-translation-orig-fr2en
+    reference: 2 previous contexts (orig language) complete translation, French-to-English
+  35aba14e-1b84-41a2-8150-e0adfd1684a2: !Template
+    answer_choices: null
+    id: 35aba14e-1b84-41a2-8150-e0adfd1684a2
+    jinja: '{% set target_lang = "english" %}{% if utterance_meta.lang == "english"
+      %}{% set target_lang = "french" %}{% endif %}
+
+      {% for previous in dialogue_history[-2:] %}
+
+      {{ previous.orig }}{% endfor %}
+
+      {{ orig }}
+
+
+      The {% if utterance_meta.lang == "english" %}French{% else %}English{% endif
+      %} translation is:
+
+      {% for previous in dialogue_history[-2:] %}{% if previous.utterance_meta.lang
+      == target_lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
+
+      {% endfor %} ||| {% if utterance_meta.lang =="french" %}{{ ref }}{% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT-1-context-complete-translation-orig-fr2en
+    reference: 1 previous context (orig language) complete translation, French-to-English
   40c1ee21-3cf4-4efc-b902-d3aab0a79c40: !Template
     answer_choices: null
     id: 40c1ee21-3cf4-4efc-b902-d3aab0a79c40
@@ -259,13 +307,11 @@ templates:
 
       {% endfor %}{% endif %}
 
-      Translate the next utterance in the dialogue into English:
+      Translate {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}''s
+      next utterance into English: {{ orig }}
 
-      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}
 
-
-      ||| {% if utterance_meta.lang == "french" %}{% if utterance_meta.lang == first_lang
-      %}A{% else %}B{% endif %}: {{ ref }}{% endif %}
+      ||| {% if utterance_meta.lang == "french" %} {{ ref }}{% endif %}
 
       '
     metadata: !TemplateMetadata
@@ -293,13 +339,11 @@ templates:
 
       {% endfor %}{% endif %}
 
-      Translate the next utterance in the dialogue into English:
-
-      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}
+      Translate {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}''s
+      next utterance into English: {{ orig }}
 
 
-      ||| {% if utterance_meta.lang == "french" %}{% if utterance_meta.lang == first_lang
-      %}A{% else %}B{% endif %}: {{ ref }}{% endif %}
+      ||| {% if utterance_meta.lang == "french" %}{{ ref }}{% endif %}
 
       '
     metadata: !TemplateMetadata
@@ -312,7 +356,7 @@ templates:
   72c1cbbd-956f-4721-9cee-fd50148cf77d: !Template
     answer_choices: null
     id: 72c1cbbd-956f-4721-9cee-fd50148cf77d
-    jinja: 'Translate the following into English: "{{ orig }}" ||| {% if utterance_meta.lang
+    jinja: 'Translate this into English: {{ orig }} ||| {% if utterance_meta.lang
       == "french"  %}{{ ref }}{% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
@@ -321,6 +365,33 @@ templates:
       original_task: true
     name: MT_sent_fr2en
     reference: Sentence-level, French-to-English
+  7c67ca68-f4bc-428e-a708-487809492fcc: !Template
+    answer_choices: null
+    id: 7c67ca68-f4bc-428e-a708-487809492fcc
+    jinja: '{% set target_lang = "english" %}{% if utterance_meta.lang == "english"
+      %}{% set target_lang = "french" %}{% endif %}
+
+      {% for previous in dialogue_history[-2:] %}
+
+      {{ previous.orig }}{% endfor %}
+
+      {{ orig }}
+
+
+      The {% if utterance_meta.lang == "english" %}French{% else %}English{% endif
+      %} translation is:
+
+      {% for previous in dialogue_history[-2:] %}{% if previous.utterance_meta.lang
+      == target_lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
+
+      {% endfor %} ||| {% if utterance_meta.lang =="english" %}{{ ref }}{% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT-2-context-complete-translation-orig-en2fr
+    reference: 2 previous contexts (orig language) complete translation, English-to-French
   836c5096-2fd6-478c-89e8-39a050347d28: !Template
     answer_choices: null
     id: 836c5096-2fd6-478c-89e8-39a050347d28
@@ -338,13 +409,11 @@ templates:
 
       {% endfor %}{% endif %}
 
-      Translate the next utterance in the dialogue into English:
-
-      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}
+      Translate {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}''s
+      next utterance in the dialogue into English: {{ orig }}
 
 
-      ||| {% if utterance_meta.lang == "french" %}{% if utterance_meta.lang == first_lang
-      %}A{% else %}B{% endif %}: {{ ref }}{% endif %}'
+      ||| {% if utterance_meta.lang == "french" %}{{ ref }}{% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -355,7 +424,7 @@ templates:
   842dc41a-8af0-4dca-8b55-a87026bfac31: !Template
     answer_choices: null
     id: 842dc41a-8af0-4dca-8b55-a87026bfac31
-    jinja: 'Translate the following into {% if utterance_meta.lang == "english"  %}French{%
+    jinja: 'Translate this into {% if utterance_meta.lang == "english"  %}French{%
       else %}English{% endif %}: "{{ orig }}"? ||| {{ ref }}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
@@ -380,14 +449,12 @@ templates:
 
       {% endfor %}{% endif %}
 
-      Translate the next utterance in the dialogue into {% if utterance_meta.lang
-      == "english" %}French{% else %}English{% endif %}:
+      Translate {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}''s
+      next utterance into {% if utterance_meta.lang == "english" %}French{% else %}English{%
+      endif %}: {{ orig }}
 
-      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}
 
-
-      ||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ ref
-      }}'
+      ||| {{ ref }}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -398,8 +465,8 @@ templates:
   96da9c0e-e606-4283-8494-7129c75b4d02: !Template
     answer_choices: null
     id: 96da9c0e-e606-4283-8494-7129c75b4d02
-    jinja: 'Translate the following into French: "{{ orig }}" ||| {% if utterance_meta.lang
-      == "english"  %}{{ ref }}{% endif %}'
+    jinja: 'Translate this into French: {{ orig }} ||| {% if utterance_meta.lang ==
+      "english"  %}{{ ref }}{% endif %}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -429,6 +496,33 @@ templates:
       original_task: true
     name: MT_1-context-analogy_lang-given_en2fr
     reference: 1 previous context (same language) used for analogy, English-to-French
+  a7511f73-dd28-449f-bc6c-9609b736bb40: !Template
+    answer_choices: null
+    id: a7511f73-dd28-449f-bc6c-9609b736bb40
+    jinja: '{% set target_lang = "english" %}{% if utterance_meta.lang == "english"
+      %}{% set target_lang = "french" %}{% endif %}
+
+      {% for previous in dialogue_history[-2:] %}
+
+      {{ previous.orig }}{% endfor %}
+
+      {{ orig }}
+
+
+      The {% if utterance_meta.lang == "english" %}French{% else %}English{% endif
+      %} translation is:
+
+      {% for previous in dialogue_history[-2:] %}{% if previous.utterance_meta.lang
+      == target_lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
+
+      {% endfor %} ||| {{ ref }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT-2-context-complete-translation-orig-both-directions
+    reference: 2 previous contexts (orig language) complete translation, both directions
   ac4c63da-32d2-40ac-aa7a-632e8ba42b4a: !Template
     answer_choices: A ||| B
     id: ac4c63da-32d2-40ac-aa7a-632e8ba42b4a
@@ -465,14 +559,12 @@ templates:
 
       {% endfor %}{% endif %}
 
-      Translate the next utterance in the dialogue into {% if utterance_meta.lang
-      == "english" %}French{% else %}English{% endif %}:
+      Translate {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}''s
+      next utterance into {% if utterance_meta.lang == "english" %}French{% else %}English{%
+      endif %}: {{ orig }}
 
-      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}
 
-
-      ||| {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ ref
-      }}'
+      ||| {{ ref }}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -480,6 +572,33 @@ templates:
       original_task: true
     name: MT_5-context_same-lang-mt_both-directions
     reference: Up to 5 previous sentences (same language, MT), both language directions
+  ca6d7d89-e32d-4993-a73d-ba7b09f1597f: !Template
+    answer_choices: null
+    id: ca6d7d89-e32d-4993-a73d-ba7b09f1597f
+    jinja: '{% set target_lang = "english" %}{% if utterance_meta.lang == "english"
+      %}{% set target_lang = "french" %}{% endif %}
+
+      {% for previous in dialogue_history[-2:] %}
+
+      {{ previous.orig }}{% endfor %}
+
+      {{ orig }}
+
+
+      The {% if utterance_meta.lang == "english" %}French{% else %}English{% endif
+      %} translation is:
+
+      {% for previous in dialogue_history[-2:] %}{% if previous.utterance_meta.lang
+      == target_lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
+
+      {% endfor %} ||| {% if utterance_meta.lang =="english" %}{{ ref }}{% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT-1-context-complete-translation-orig-en2fr
+    reference: 1 previous context (orig language) complete translation, English-to-French
   d068ba77-ce75-4de3-a170-ca0a7cd3f8d1: !Template
     answer_choices: null
     id: d068ba77-ce75-4de3-a170-ca0a7cd3f8d1
@@ -497,13 +616,11 @@ templates:
 
       {% endfor %}{% endif %}
 
-      Translate the next utterance in the dialogue into French:
+      Translate {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}''s
+      next utterance into French: {{ orig }}
 
-      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ orig }}
 
-
-      ||| {% if utterance_meta.lang == "english" %}{% if utterance_meta.lang == first_lang
-      %}A{% else %}B{% endif %}: {{ ref }}{% endif %}
+      ||| {% if utterance_meta.lang == "english" %}{{ ref }}{% endif %}
 
       '
     metadata: !TemplateMetadata
@@ -546,8 +663,8 @@ templates:
       or {{ "no" }}?
 
 
-      ||| {% if utterance_meta.eval_problems|length > 0 %}{% if utterance_meta.lang
-      == "french" %}{{ ["yes", "no" ][reply] }}{% endif %}{% endif %}
+      ||| {% if utterance_meta.lang == "french" %}{{ ["yes", "no" ][reply] }}{% endif
+      %}
 
       '
     metadata: !TemplateMetadata
@@ -558,6 +675,33 @@ templates:
     name: classify-errors_same-lang_en
     reference: Identify presence of notable errors, same language context (5), English
       translation
+  e9f5dfda-42b5-4698-8e8d-9fc9924e2e29: !Template
+    answer_choices: null
+    id: e9f5dfda-42b5-4698-8e8d-9fc9924e2e29
+    jinja: '{% set target_lang = "english" %}{% if utterance_meta.lang == "english"
+      %}{% set target_lang = "french" %}{% endif %}
+
+      {% for previous in dialogue_history[-1:] %}
+
+      {{ previous.orig }}{% endfor %}
+
+      {{ orig }}
+
+
+      The {% if utterance_meta.lang == "english" %}French{% else %}English{% endif
+      %} translation is:
+
+      {% for previous in dialogue_history[-1:] %}{% if previous.utterance_meta.lang
+      == target_lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
+
+      {% endfor %} ||| {{ ref }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      original_task: true
+    name: MT-1-context-complete-translation-orig-both-directions
+    reference: 1 previous context (orig language) complete translation, both directions
   eea8f47e-9bf5-4423-980b-58a9635c1f49: !Template
     answer_choices: null
     id: eea8f47e-9bf5-4423-980b-58a9635c1f49

From 2728caf98da70c74fb008e9f3018eabbe91c557e Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Wed, 11 May 2022 17:47:08 +0200
Subject: [PATCH 14/21] corrected duplicate name

---
 promptsource/templates/rbawden/DiaBLa/templates.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index 19243b92f..abaa6e832 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -21,7 +21,7 @@ templates:
       - BLEU
       original_task: true
     name: MT_1-context-analogy_lang-given_fr2en
-    reference: 1 previous context (same language) used for analogy, French-to-English
+    reference: 1 previous context (same language) used for analogy, lang given, French-to-English
   0e841022-eb3c-4f5a-b57a-0cbd3d628b68: !Template
     answer_choices: null
     id: 0e841022-eb3c-4f5a-b57a-0cbd3d628b68
@@ -289,7 +289,8 @@ templates:
       - BLEU
       original_task: true
     name: MT_1-context-analogy_lang-given_both-directions
-    reference: 1 previous context (same language) used for analogy, both directions
+    reference: 1 previous context (same language) used for analogy, language given,
+      both directions
   702f1f50-d6b0-4e65-a8e6-b449257f1e35: !Template
     answer_choices: null
     id: 702f1f50-d6b0-4e65-a8e6-b449257f1e35
@@ -495,7 +496,7 @@ templates:
       - BLEU
       original_task: true
     name: MT_1-context-analogy_lang-given_en2fr
-    reference: 1 previous context (same language) used for analogy, English-to-French
+    reference: 1 previous context (same language) used for analogy, lang given, English-to-French
   a7511f73-dd28-449f-bc6c-9609b736bb40: !Template
     answer_choices: null
     id: a7511f73-dd28-449f-bc6c-9609b736bb40

From d358fe147df71b195a3c27d529301650e499c7ae Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Wed, 11 May 2022 18:01:39 +0200
Subject: [PATCH 15/21] updated duplicate definition

---
 promptsource/templates/rbawden/DiaBLa/templates.yaml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index abaa6e832..0062a9f9d 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -200,8 +200,7 @@ templates:
       {{ orig }}
 
 
-      The {% if utterance_meta.lang == "english" %}French{% else %}English{% endif
-      %} translation is:
+      The English translation is:
 
       {% for previous in dialogue_history[-2:] %}{% if previous.utterance_meta.lang
       == target_lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
@@ -379,8 +378,7 @@ templates:
       {{ orig }}
 
 
-      The {% if utterance_meta.lang == "english" %}French{% else %}English{% endif
-      %} translation is:
+      The French translation is:
 
       {% for previous in dialogue_history[-2:] %}{% if previous.utterance_meta.lang
       == target_lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}

From da98d593f75d044eb3a5927920d887030bb32d94 Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Thu, 12 May 2022 09:57:00 +0200
Subject: [PATCH 16/21] corrected error of only two pipes in answer choices

---
 .../templates/rbawden/DiaBLa/templates.yaml   | 37 ++++++++++++++-----
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index 0062a9f9d..07bfcccdd 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -51,7 +51,7 @@ templates:
     name: MT_5-context_orig-lang_en2fr
     reference: Up to 5 previous sentences (original language), English-to-French
   123f36ee-c808-4cf0-aa5c-ed1f03ac3b69: !Template
-    answer_choices: yes || no
+    answer_choices: yes ||| no
     id: 123f36ee-c808-4cf0-aa5c-ed1f03ac3b69
     jinja: '{% set options = ["word choice", "grammar", "style", "coherence", "meaning"]
       %}
@@ -144,7 +144,7 @@ templates:
     name: MT_5-context_same-lang-ref_both-directions
     reference: Up to 5 previous sentences (same language, ref), both language directions
   28ea04f4-338e-40cf-8730-4a794b5b64b2: !Template
-    answer_choices: yes || no
+    answer_choices: yes ||| no
     id: 28ea04f4-338e-40cf-8730-4a794b5b64b2
     jinja: "{% set options = [\"word choice\", \"grammar\", \"style\", \"coherence\"\
       , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% set reply=0 %}\n\
@@ -226,8 +226,7 @@ templates:
       {{ orig }}
 
 
-      The {% if utterance_meta.lang == "english" %}French{% else %}English{% endif
-      %} translation is:
+      The English translation is:
 
       {% for previous in dialogue_history[-2:] %}{% if previous.utterance_meta.lang
       == target_lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
@@ -525,7 +524,7 @@ templates:
   ac4c63da-32d2-40ac-aa7a-632e8ba42b4a: !Template
     answer_choices: A ||| B
     id: ac4c63da-32d2-40ac-aa7a-632e8ba42b4a
-    jinja: '{% set label = range(0,2)|random %}
+    jinja: '{% set label = 0 %}
 
       Which of the following translations of "{{ orig }}" is produced automatically?
 
@@ -539,8 +538,8 @@ templates:
       metrics:
       - Accuracy
       original_task: false
-    name: discriminate-mt-ref
-    reference: Identify-MT-output
+    name: discriminate-mt-ref-A
+    reference: Identify-MT-output, answer=A
   b61c81ec-29eb-47f8-a1c6-561264ac04f3: !Template
     answer_choices: null
     id: b61c81ec-29eb-47f8-a1c6-561264ac04f3
@@ -584,8 +583,7 @@ templates:
       {{ orig }}
 
 
-      The {% if utterance_meta.lang == "english" %}French{% else %}English{% endif
-      %} translation is:
+      The French translation is:
 
       {% for previous in dialogue_history[-2:] %}{% if previous.utterance_meta.lang
       == target_lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
@@ -598,6 +596,25 @@ templates:
       original_task: true
     name: MT-1-context-complete-translation-orig-en2fr
     reference: 1 previous context (orig language) complete translation, English-to-French
+  cfed1937-b4a5-4100-b489-3d8d9cd84423: !Template
+    answer_choices: A ||| B
+    id: cfed1937-b4a5-4100-b489-3d8d9cd84423
+    jinja: '{% set label = 1 %}
+
+      Which of the following translations of "{{ orig }}" is produced automatically?
+
+      {{ "A" }}) {% if label == 0 %}{{ ref }}{% else %}{{ mt }}{% endif %}
+
+      {{ "B" }}) {% if label == 0 %}{{ mt }}{% else %}{{ ref }}{% endif %}
+
+      ||| {% if ref != mt %}{{ ["A", "B"][label] }}{% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
+      original_task: false
+    name: discriminate-mt-ref-B
+    reference: Identify-MT-output, answer=B
   d068ba77-ce75-4de3-a170-ca0a7cd3f8d1: !Template
     answer_choices: null
     id: d068ba77-ce75-4de3-a170-ca0a7cd3f8d1
@@ -630,7 +647,7 @@ templates:
     name: MT_5-context_same-lang-mt_en2fr
     reference: Up to 5 previous sentences (same language, MT), English-to-French
   dddd6f35-5a01-431b-b59b-e9a8e76377aa: !Template
-    answer_choices: yes || no
+    answer_choices: yes ||| no
     id: dddd6f35-5a01-431b-b59b-e9a8e76377aa
     jinja: '{% set options = ["word choice", "grammar", "style", "coherence", "meaning"]
       %}

From 54abd8ac3b6944a63da91f7113533d33c670d4b0 Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Thu, 12 May 2022 10:05:41 +0200
Subject: [PATCH 17/21] corrected -2 index to -1 - duplicate defintiions

---
 promptsource/templates/rbawden/DiaBLa/templates.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index 07bfcccdd..2a34458bc 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -219,7 +219,7 @@ templates:
     jinja: '{% set target_lang = "english" %}{% if utterance_meta.lang == "english"
       %}{% set target_lang = "french" %}{% endif %}
 
-      {% for previous in dialogue_history[-2:] %}
+      {% for previous in dialogue_history[-1:] %}
 
       {{ previous.orig }}{% endfor %}
 
@@ -228,7 +228,7 @@ templates:
 
       The English translation is:
 
-      {% for previous in dialogue_history[-2:] %}{% if previous.utterance_meta.lang
+      {% for previous in dialogue_history[-1:] %}{% if previous.utterance_meta.lang
       == target_lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
 
       {% endfor %} ||| {% if utterance_meta.lang =="french" %}{{ ref }}{% endif %}'
@@ -576,7 +576,7 @@ templates:
     jinja: '{% set target_lang = "english" %}{% if utterance_meta.lang == "english"
       %}{% set target_lang = "french" %}{% endif %}
 
-      {% for previous in dialogue_history[-2:] %}
+      {% for previous in dialogue_history[-1:] %}
 
       {{ previous.orig }}{% endfor %}
 
@@ -585,7 +585,7 @@ templates:
 
       The French translation is:
 
-      {% for previous in dialogue_history[-2:] %}{% if previous.utterance_meta.lang
+      {% for previous in dialogue_history[-1:] %}{% if previous.utterance_meta.lang
       == target_lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
 
       {% endfor %} ||| {% if utterance_meta.lang =="english" %}{{ ref }}{% endif %}'

From f2400206b1ae514fb81f02f3bf6953f4c748c3de Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Fri, 13 May 2022 18:22:51 +0200
Subject: [PATCH 18/21] merged with eval-hackathon updates

---
 promptsource/templates.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/promptsource/templates.py b/promptsource/templates.py
index 8a54a089a..a4bb46a58 100644
--- a/promptsource/templates.py
+++ b/promptsource/templates.py
@@ -30,7 +30,6 @@
 INCLUDED_USERS = {"Zaid", "craffel", "GEM", "aps", "rbawden"}
 
 
-
 def highlight(input):
     return "<span style='color: #F08080'>" + input + "</span>"
 

From 26419f9aa0f9f588693b8e09a0428b5b193f6c7a Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Mon, 16 May 2022 10:47:23 +0200
Subject: [PATCH 19/21] corrected discriminate mt ref

---
 .../templates/rbawden/DiaBLa/templates.yaml   | 20 ++++++++-----------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index 2a34458bc..41d19c846 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -524,15 +524,13 @@ templates:
   ac4c63da-32d2-40ac-aa7a-632e8ba42b4a: !Template
     answer_choices: A ||| B
     id: ac4c63da-32d2-40ac-aa7a-632e8ba42b4a
-    jinja: '{% set label = 0 %}
+    jinja: 'Which of the following translations of "{{ orig }}" is produced automatically?
 
-      Which of the following translations of "{{ orig }}" is produced automatically?
+      {{ "A" }}) {{ mt }}
 
-      {{ "A" }}) {% if label == 0 %}{{ ref }}{% else %}{{ mt }}{% endif %}
+      {{ "B" }}) {{ ref }}
 
-      {{ "B" }}) {% if label == 0 %}{{ mt }}{% else %}{{ ref }}{% endif %}
-
-      ||| {% if ref != mt %}{{ ["A", "B"][label] }}{% endif %}'
+      |||{{ ["A", "B"][0] }}'
     metadata: !TemplateMetadata
       choices_in_prompt: true
       metrics:
@@ -599,15 +597,13 @@ templates:
   cfed1937-b4a5-4100-b489-3d8d9cd84423: !Template
     answer_choices: A ||| B
     id: cfed1937-b4a5-4100-b489-3d8d9cd84423
-    jinja: '{% set label = 1 %}
-
-      Which of the following translations of "{{ orig }}" is produced automatically?
+    jinja: 'Which of the following translations of "{{ orig }}" is produced automatically?
 
-      {{ "A" }}) {% if label == 0 %}{{ ref }}{% else %}{{ mt }}{% endif %}
+      {{ "A" }}) {{ ref }}
 
-      {{ "B" }}) {% if label == 0 %}{{ mt }}{% else %}{{ ref }}{% endif %}
+      {{ "B" }}) {{ mt }}
 
-      ||| {% if ref != mt %}{{ ["A", "B"][label] }}{% endif %}'
+      |||{{ ["A", "B"][1] }}'
     metadata: !TemplateMetadata
       choices_in_prompt: true
       metrics:

From 013bb630490bb8a6a72e14711be7432cc5eb55f8 Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Tue, 17 May 2022 17:30:23 +0200
Subject: [PATCH 20/21] removed directional templates and only keep both
 directions (analysis to be done separately later. Also some slight modifs
 such as removing excess words and quotes

---
 .../templates/rbawden/DiaBLa/templates.yaml   | 494 +-----------------
 1 file changed, 4 insertions(+), 490 deletions(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index 41d19c846..34c3c3792 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -1,129 +1,5 @@
 dataset: rbawden/DiaBLa
 templates:
-  0e55b858-92b0-4e5b-923b-0d23518035e3: !Template
-    answer_choices: null
-    id: 0e55b858-92b0-4e5b-923b-0d23518035e3
-    jinja: '{% set previous_ref = "" %}
-
-      {% if dialogue_history|length > 0 %}
-
-      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
-      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
-      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
-      %}{% endif %}" translates into French as: {{ previous_ref }}{% endif %}
-
-
-      "{{ orig }}" translates into English as: ||| {% if utterance_meta.lang == "french"
-      %}{{ ref }}{% endif %}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      metrics:
-      - BLEU
-      original_task: true
-    name: MT_1-context-analogy_lang-given_fr2en
-    reference: 1 previous context (same language) used for analogy, lang given, French-to-English
-  0e841022-eb3c-4f5a-b57a-0cbd3d628b68: !Template
-    answer_choices: null
-    id: 0e841022-eb3c-4f5a-b57a-0cbd3d628b68
-    jinja: '{% set first_lang="" %}
-
-      {% if dialogue_history|length > 0 %}
-
-      Given the following dialogue between person A and person B:
-
-
-      {% set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous
-      in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang
-      %}A{% else %}B{% endif %}: {{ previous.orig }}
-
-      {% endfor %}{% endif %}
-
-      Translate {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}''s
-      next utterance into French: {{ orig }}
-
-
-      ||| {% if utterance_meta.lang == "english" %}{{ ref }}{% endif %}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      metrics:
-      - BLEU
-      original_task: true
-    name: MT_5-context_orig-lang_en2fr
-    reference: Up to 5 previous sentences (original language), English-to-French
-  123f36ee-c808-4cf0-aa5c-ed1f03ac3b69: !Template
-    answer_choices: yes ||| no
-    id: 123f36ee-c808-4cf0-aa5c-ed1f03ac3b69
-    jinja: '{% set options = ["word choice", "grammar", "style", "coherence", "meaning"]
-      %}
-
-      {% set label = range(0,5)|random %}
-
-      {% set reply=0 %}
-
-      {% set first_lang="" %}
-
-
-      {% if options[label] in utterance_meta.eval_problems %}{% set reply=0 %}{% else
-      %}{% set reply=1 %}{% endif %}
-
-
-      Given the following dialogue between person A and person B:
-
-      {% if dialogue_history|length > 0 %}{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang
-      %}{% for previous in dialogue_history[-5:] %}
-
-      {% if previous.utterance_meta.lang == first_lang %}A{% else %}B{% endif %}:
-      {% if previous.utterance_meta.lang != utterance_meta.lang %}{{ previous.orig
-      }}{% else %}{{ previous.mt }}{% endif %}{% endfor %}{% endif %}
-
-      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ mt }}
-
-
-      Does the last utterance contain a {{ options[label] }} problem, {{ "yes" }}
-      or {{ "no" }}?
-
-
-      ||| {% if utterance_meta.lang == "english" %}{{ ["yes", "no" ][reply] }}{% endif
-      %}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      metrics:
-      - Accuracy
-      original_task: false
-    name: classify-errors_same-lang_fr
-    reference: Identify presence of notable errors, same language context (5), French
-      translation
-  1798dee9-8a9c-45ed-87d8-237ec7dbf5ca: !Template
-    answer_choices: null
-    id: 1798dee9-8a9c-45ed-87d8-237ec7dbf5ca
-    jinja: '{% set first_lang="" %}
-
-      {% if dialogue_history|length > 0 %}
-
-      Given the following dialogue between person A and person B:
-
-
-      {% set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous
-      in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang
-      %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang
-      %}{{ previous.orig }}{% else %}{{ previous.ref }}{% endif %}
-
-      {% endfor %}{% endif %}
-
-      Translate {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}''s
-      next utterance into French: {{ orig }}
-
-
-      ||| {% if utterance_meta.lang == "english" %} {{ ref }}{% endif %}
-
-      '
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      metrics:
-      - BLEU
-      original_task: true
-    name: MT_5-context_same-lang-ref_en2fr
-    reference: Up to 5 previous sentences (same language, ref), English-to-French
   2731216a-b994-48f9-aaf6-00c7038bbed5: !Template
     answer_choices: null
     id: 2731216a-b994-48f9-aaf6-00c7038bbed5
@@ -133,9 +9,9 @@ templates:
       \ == first_lang %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang\
       \ == utterance_meta.lang %}{{ previous.orig }}{% else %}{{ previous.ref }}{%\
       \ endif %}\n{% endfor %}{% endif %} \nTranslate {% if utterance_meta.lang ==\
-      \ first_lang %}A{% else %}B{% endif %}'s next utterance in the dialogue into\
-      \ {% if utterance_meta.lang == \"english\" %}French{% else %}English{% endif\
-      \ %}: {{ orig }}\n\n||| {{ ref }}"
+      \ first_lang %}A{% else %}B{% endif %}'s next utterance into {% if utterance_meta.lang\
+      \ == \"english\" %}French{% else %}English{% endif %}: {{ orig }}\n\n||| {{\
+      \ ref }}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -165,102 +41,6 @@ templates:
       original_task: false
     name: classify-errors_same-lang_both-directions
     reference: Identify presence of notable errors
-  2abc728f-e7fc-44fc-ba2a-ad78ae2469a9: !Template
-    answer_choices: null
-    id: 2abc728f-e7fc-44fc-ba2a-ad78ae2469a9
-    jinja: '{% set previous_ref = "" %}
-
-      {% if dialogue_history|length > 0 %}
-
-      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
-      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
-      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
-      %}{% endif %}" translates as: {{ previous_ref }}{% endif %}
-
-
-      "{{ orig }}" translates as: ||| {% if dialogue_history|length > 0 %}{% if utterance_meta.lang
-      == "english" %}{{ ref }}{% endif %}{% endif %}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      metrics:
-      - BLEU
-      original_task: true
-    name: MT_1-context-analogy_infer-lang_en2fr
-    reference: 1 previous context (same language) used for analogy, English-to-French
-  35822dda-6f09-418a-a2a3-2178d94203de: !Template
-    answer_choices: null
-    id: 35822dda-6f09-418a-a2a3-2178d94203de
-    jinja: '{% set target_lang = "english" %}{% if utterance_meta.lang == "english"
-      %}{% set target_lang = "french" %}{% endif %}
-
-      {% for previous in dialogue_history[-2:] %}
-
-      {{ previous.orig }}{% endfor %}
-
-      {{ orig }}
-
-
-      The English translation is:
-
-      {% for previous in dialogue_history[-2:] %}{% if previous.utterance_meta.lang
-      == target_lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
-
-      {% endfor %} ||| {% if utterance_meta.lang =="french" %}{{ ref }}{% endif %}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      metrics:
-      - BLEU
-      original_task: true
-    name: MT-2-context-complete-translation-orig-fr2en
-    reference: 2 previous contexts (orig language) complete translation, French-to-English
-  35aba14e-1b84-41a2-8150-e0adfd1684a2: !Template
-    answer_choices: null
-    id: 35aba14e-1b84-41a2-8150-e0adfd1684a2
-    jinja: '{% set target_lang = "english" %}{% if utterance_meta.lang == "english"
-      %}{% set target_lang = "french" %}{% endif %}
-
-      {% for previous in dialogue_history[-1:] %}
-
-      {{ previous.orig }}{% endfor %}
-
-      {{ orig }}
-
-
-      The English translation is:
-
-      {% for previous in dialogue_history[-1:] %}{% if previous.utterance_meta.lang
-      == target_lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
-
-      {% endfor %} ||| {% if utterance_meta.lang =="french" %}{{ ref }}{% endif %}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      metrics:
-      - BLEU
-      original_task: true
-    name: MT-1-context-complete-translation-orig-fr2en
-    reference: 1 previous context (orig language) complete translation, French-to-English
-  40c1ee21-3cf4-4efc-b902-d3aab0a79c40: !Template
-    answer_choices: null
-    id: 40c1ee21-3cf4-4efc-b902-d3aab0a79c40
-    jinja: '{% set previous_ref = "" %}
-
-      {% if dialogue_history|length > 0 %}
-
-      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
-      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
-      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
-      %}{% endif %}" translates as: {{ previous_ref }}{% endif %}
-
-
-      "{{ orig }}" translates as: ||| {% if utterance_meta.lang == "french" %}{{ ref
-      }}{% endif %}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      metrics:
-      - BLEU
-      original_task: true
-    name: MT_1-context-analogy_infer-lang_fr2en
-    reference: 1 previous context (same language) used for analogy, French-to-English
   6a01fbe6-d5ec-4ad9-a2ee-3c48ed095885: !Template
     answer_choices: null
     id: 6a01fbe6-d5ec-4ad9-a2ee-3c48ed095885
@@ -289,141 +69,11 @@ templates:
     name: MT_1-context-analogy_lang-given_both-directions
     reference: 1 previous context (same language) used for analogy, language given,
       both directions
-  702f1f50-d6b0-4e65-a8e6-b449257f1e35: !Template
-    answer_choices: null
-    id: 702f1f50-d6b0-4e65-a8e6-b449257f1e35
-    jinja: '{% set first_lang="" %}
-
-      {% if dialogue_history|length > 0 %}
-
-      Given the following dialogue between person A and person B:
-
-
-      {% set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous
-      in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang
-      %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang
-      %}{{ previous.orig }}{% else %}{{ previous.ref }}{% endif %}
-
-      {% endfor %}{% endif %}
-
-      Translate {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}''s
-      next utterance into English: {{ orig }}
-
-
-      ||| {% if utterance_meta.lang == "french" %} {{ ref }}{% endif %}
-
-      '
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      metrics:
-      - BLEU
-      original_task: true
-    name: MT_5-context_same-lang-ref_fr2en
-    reference: Up to 5 previous sentences (same language, ref), French-to-English
-  72759632-7764-4493-9a6e-8f1c8b337bde: !Template
-    answer_choices: null
-    id: 72759632-7764-4493-9a6e-8f1c8b337bde
-    jinja: '{% set first_lang="" %}
-
-
-      {% if dialogue_history|length > 0 %}
-
-      Given the following dialogue between person A and person B:
-
-
-      {% set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous
-      in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang
-      %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang
-      %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
-
-      {% endfor %}{% endif %}
-
-      Translate {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}''s
-      next utterance into English: {{ orig }}
-
-
-      ||| {% if utterance_meta.lang == "french" %}{{ ref }}{% endif %}
-
-      '
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      metrics:
-      - BLEU
-      original_task: true
-    name: MT_5-context_same-lang-mt_fr2en
-    reference: Up to 5 previous sentences (same language, MT), French-to-English
-  72c1cbbd-956f-4721-9cee-fd50148cf77d: !Template
-    answer_choices: null
-    id: 72c1cbbd-956f-4721-9cee-fd50148cf77d
-    jinja: 'Translate this into English: {{ orig }} ||| {% if utterance_meta.lang
-      == "french"  %}{{ ref }}{% endif %}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      metrics:
-      - BLEU
-      original_task: true
-    name: MT_sent_fr2en
-    reference: Sentence-level, French-to-English
-  7c67ca68-f4bc-428e-a708-487809492fcc: !Template
-    answer_choices: null
-    id: 7c67ca68-f4bc-428e-a708-487809492fcc
-    jinja: '{% set target_lang = "english" %}{% if utterance_meta.lang == "english"
-      %}{% set target_lang = "french" %}{% endif %}
-
-      {% for previous in dialogue_history[-2:] %}
-
-      {{ previous.orig }}{% endfor %}
-
-      {{ orig }}
-
-
-      The French translation is:
-
-      {% for previous in dialogue_history[-2:] %}{% if previous.utterance_meta.lang
-      == target_lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
-
-      {% endfor %} ||| {% if utterance_meta.lang =="english" %}{{ ref }}{% endif %}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      metrics:
-      - BLEU
-      original_task: true
-    name: MT-2-context-complete-translation-orig-en2fr
-    reference: 2 previous contexts (orig language) complete translation, English-to-French
-  836c5096-2fd6-478c-89e8-39a050347d28: !Template
-    answer_choices: null
-    id: 836c5096-2fd6-478c-89e8-39a050347d28
-    jinja: '{% set first_lang="" %}
-
-
-      {% if dialogue_history|length > 0 %}
-
-      Given the following dialogue between person A and person B:
-
-
-      {% set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous
-      in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang
-      %}A{% else %}B{% endif %}: {{ previous.orig }}
-
-      {% endfor %}{% endif %}
-
-      Translate {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}''s
-      next utterance in the dialogue into English: {{ orig }}
-
-
-      ||| {% if utterance_meta.lang == "french" %}{{ ref }}{% endif %}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      metrics:
-      - BLEU
-      original_task: false
-    name: MT_5-context_orig-lang_fr2en
-    reference: Up to 5 previous sentences (original language), French-to-English
   842dc41a-8af0-4dca-8b55-a87026bfac31: !Template
     answer_choices: null
     id: 842dc41a-8af0-4dca-8b55-a87026bfac31
     jinja: 'Translate this into {% if utterance_meta.lang == "english"  %}French{%
-      else %}English{% endif %}: "{{ orig }}"? ||| {{ ref }}'
+      else %}English{% endif %}: {{ orig }} ||| {{ ref }}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
@@ -460,40 +110,6 @@ templates:
       original_task: true
     name: MT_5-context_orig-lang_both-directions
     reference: Up to 5 previous sentences (original language), both language directions
-  96da9c0e-e606-4283-8494-7129c75b4d02: !Template
-    answer_choices: null
-    id: 96da9c0e-e606-4283-8494-7129c75b4d02
-    jinja: 'Translate this into French: {{ orig }} ||| {% if utterance_meta.lang ==
-      "english"  %}{{ ref }}{% endif %}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      metrics:
-      - BLEU
-      original_task: true
-    name: MT_sent_en2fr
-    reference: Sentence-level, English-to-French
-  97d56019-a7df-4733-aed0-332c222e9079: !Template
-    answer_choices: null
-    id: 97d56019-a7df-4733-aed0-332c222e9079
-    jinja: '{% set previous_ref = "" %}
-
-      {% if dialogue_history|length > 0 %}
-
-      "{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{{
-      dialogue_history[-1].orig }}{% set previous_ref = dialogue_history[-1].ref %}{%
-      else %}{{ dialogue_history[-1].ref }}{% set previous_ref = dialogue_history[-1].orig
-      %}{% endif %}" translates into French as: {{ previous_ref }}{% endif %}
-
-
-      "{{ orig }}" translates into French as: ||| {% if utterance_meta.lang == "english"
-      %}{{ ref }}{% endif %}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      metrics:
-      - BLEU
-      original_task: true
-    name: MT_1-context-analogy_lang-given_en2fr
-    reference: 1 previous context (same language) used for analogy, lang given, English-to-French
   a7511f73-dd28-449f-bc6c-9609b736bb40: !Template
     answer_choices: null
     id: a7511f73-dd28-449f-bc6c-9609b736bb40
@@ -568,32 +184,6 @@ templates:
       original_task: true
     name: MT_5-context_same-lang-mt_both-directions
     reference: Up to 5 previous sentences (same language, MT), both language directions
-  ca6d7d89-e32d-4993-a73d-ba7b09f1597f: !Template
-    answer_choices: null
-    id: ca6d7d89-e32d-4993-a73d-ba7b09f1597f
-    jinja: '{% set target_lang = "english" %}{% if utterance_meta.lang == "english"
-      %}{% set target_lang = "french" %}{% endif %}
-
-      {% for previous in dialogue_history[-1:] %}
-
-      {{ previous.orig }}{% endfor %}
-
-      {{ orig }}
-
-
-      The French translation is:
-
-      {% for previous in dialogue_history[-1:] %}{% if previous.utterance_meta.lang
-      == target_lang %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
-
-      {% endfor %} ||| {% if utterance_meta.lang =="english" %}{{ ref }}{% endif %}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      metrics:
-      - BLEU
-      original_task: true
-    name: MT-1-context-complete-translation-orig-en2fr
-    reference: 1 previous context (orig language) complete translation, English-to-French
   cfed1937-b4a5-4100-b489-3d8d9cd84423: !Template
     answer_choices: A ||| B
     id: cfed1937-b4a5-4100-b489-3d8d9cd84423
@@ -611,82 +201,6 @@ templates:
       original_task: false
     name: discriminate-mt-ref-B
     reference: Identify-MT-output, answer=B
-  d068ba77-ce75-4de3-a170-ca0a7cd3f8d1: !Template
-    answer_choices: null
-    id: d068ba77-ce75-4de3-a170-ca0a7cd3f8d1
-    jinja: '{% set first_lang="" %}
-
-      {% if dialogue_history|length > 0 %}
-
-      Given the following dialogue between person A and person B:
-
-
-      {% set first_lang=dialogue_history[-5:][0].utterance_meta.lang %}{% for previous
-      in dialogue_history[-5:] %}{% if previous.utterance_meta.lang == first_lang
-      %}A{% else %}B{% endif %}: {% if previous.utterance_meta.lang == utterance_meta.lang
-      %}{{ previous.orig }}{% else %}{{ previous.mt }}{% endif %}
-
-      {% endfor %}{% endif %}
-
-      Translate {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}''s
-      next utterance into French: {{ orig }}
-
-
-      ||| {% if utterance_meta.lang == "english" %}{{ ref }}{% endif %}
-
-      '
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      metrics:
-      - BLEU
-      original_task: true
-    name: MT_5-context_same-lang-mt_en2fr
-    reference: Up to 5 previous sentences (same language, MT), English-to-French
-  dddd6f35-5a01-431b-b59b-e9a8e76377aa: !Template
-    answer_choices: yes ||| no
-    id: dddd6f35-5a01-431b-b59b-e9a8e76377aa
-    jinja: '{% set options = ["word choice", "grammar", "style", "coherence", "meaning"]
-      %}
-
-      {% set label = range(0,5)|random %}
-
-      {% set reply=0 %}
-
-      {% set first_lang="" %}
-
-
-      {% if options[label] in utterance_meta.eval_problems %}{% set reply=0 %}{% else
-      %}{% set reply=1 %}{% endif %}
-
-
-      Given the following dialogue between person A and person B:
-
-      {% if dialogue_history|length > 0 %}{% set first_lang=dialogue_history[-5:][0].utterance_meta.lang
-      %}{% for previous in dialogue_history[-5:] %}
-
-      {% if previous.utterance_meta.lang == first_lang %}A{% else %}B{% endif %}:
-      {% if previous.utterance_meta.lang != utterance_meta.lang %}{{ previous.orig
-      }}{% else %}{{ previous.mt }}{% endif %}{% endfor %}{% endif %}
-
-      {% if utterance_meta.lang == first_lang %}A{% else %}B{% endif %}: {{ mt }}
-
-
-      Does the last utterance contain a {{ options[label] }} problem, {{ "yes" }}
-      or {{ "no" }}?
-
-
-      ||| {% if utterance_meta.lang == "french" %}{{ ["yes", "no" ][reply] }}{% endif
-      %}
-
-      '
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      metrics:
-      - Accuracy
-      original_task: false
-    name: classify-errors_same-lang_en
-    reference: Identify presence of notable errors, same language context (5), English
-      translation
   e9f5dfda-42b5-4698-8e8d-9fc9924e2e29: !Template
     answer_choices: null
     id: e9f5dfda-42b5-4698-8e8d-9fc9924e2e29

From 7b73d78d98fc90dc5dcfdcfe9901ee1a7b245bf8 Mon Sep 17 00:00:00 2001
From: Rachel Bawden <rachel.bawden@inria.fr@users.noreply.github.com>
Date: Tue, 17 May 2022 22:57:17 +0200
Subject: [PATCH 21/21] simplified names, changed random to choice

---
 .../templates/rbawden/DiaBLa/templates.yaml   | 77 ++++++++-----------
 1 file changed, 33 insertions(+), 44 deletions(-)

diff --git a/promptsource/templates/rbawden/DiaBLa/templates.yaml b/promptsource/templates/rbawden/DiaBLa/templates.yaml
index 34c3c3792..98834e54c 100644
--- a/promptsource/templates/rbawden/DiaBLa/templates.yaml
+++ b/promptsource/templates/rbawden/DiaBLa/templates.yaml
@@ -17,13 +17,14 @@ templates:
       metrics:
       - BLEU
       original_task: true
-    name: MT_5-context_same-lang-ref_both-directions
-    reference: Up to 5 previous sentences (same language, ref), both language directions
+    name: MT with dialogue context (5 same context ref)
+    reference: MT with dialogue context (up to 5 previous sentences, same language
+      ref)
   28ea04f4-338e-40cf-8730-4a794b5b64b2: !Template
     answer_choices: yes ||| no
     id: 28ea04f4-338e-40cf-8730-4a794b5b64b2
     jinja: "{% set options = [\"word choice\", \"grammar\", \"style\", \"coherence\"\
-      , \"meaning\"] %}\n{% set label = range(0,5)|random %}\n{% set reply=0 %}\n\
+      , \"meaning\"] %}\n{% set label = range(0,5)|choice %}\n{% set reply=0 %}\n\
       {% set first_lang=\"\" %}\n{% if options[label] in utterance_meta.eval_problems\
       \ %}{% set reply=0 %}{% else %}{% set reply=1 %}{% endif %}\n{% if dialogue_history|length\
       \ > 0 %}\nGiven the following dialogue between person A and person B:\n\n{%\
@@ -39,8 +40,9 @@ templates:
       metrics:
       - Accuracy
       original_task: false
-    name: classify-errors_same-lang_both-directions
-    reference: Identify presence of notable errors
+    name: Is the error present? (same lang)
+    reference: Identify presence of notable errors given 5 context sentences (same
+      lang mt)
   6a01fbe6-d5ec-4ad9-a2ee-3c48ed095885: !Template
     answer_choices: null
     id: 6a01fbe6-d5ec-4ad9-a2ee-3c48ed095885
@@ -66,9 +68,8 @@ templates:
       metrics:
       - BLEU
       original_task: true
-    name: MT_1-context-analogy_lang-given_both-directions
-    reference: 1 previous context (same language) used for analogy, language given,
-      both directions
+    name: MT analogy, lang given (1 same context)
+    reference: MT task, translation by analogy (same language), target language provided
   842dc41a-8af0-4dca-8b55-a87026bfac31: !Template
     answer_choices: null
     id: 842dc41a-8af0-4dca-8b55-a87026bfac31
@@ -79,8 +80,8 @@ templates:
       metrics:
       - BLEU
       original_task: true
-    name: MT_sent_both-directions
-    reference: Sentence-level, both directions
+    name: MT sentence-level
+    reference: Sentence-level
   93f5256d-bd93-4056-b466-152b55860d02: !Template
     answer_choices: null
     id: 93f5256d-bd93-4056-b466-152b55860d02
@@ -108,8 +109,8 @@ templates:
       metrics:
       - BLEU
       original_task: true
-    name: MT_5-context_orig-lang_both-directions
-    reference: Up to 5 previous sentences (original language), both language directions
+    name: MT with dialogue context (5 orig context)
+    reference: MT with dialogue context (up to 5 previous sentences, original language)
   a7511f73-dd28-449f-bc6c-9609b736bb40: !Template
     answer_choices: null
     id: a7511f73-dd28-449f-bc6c-9609b736bb40
@@ -131,29 +132,32 @@ templates:
 
       {% endfor %} ||| {{ ref }}'
     metadata: !TemplateMetadata
-      choices_in_prompt: true
+      choices_in_prompt: false
       metrics:
       - BLEU
       original_task: true
-    name: MT-2-context-complete-translation-orig-both-directions
-    reference: 2 previous contexts (orig language) complete translation, both directions
+    name: MT complete translation (2 orig context)
+    reference: MT task, complete the translation, 2 sentences of previous context
+      (orig language)
   ac4c63da-32d2-40ac-aa7a-632e8ba42b4a: !Template
     answer_choices: A ||| B
     id: ac4c63da-32d2-40ac-aa7a-632e8ba42b4a
-    jinja: 'Which of the following translations of "{{ orig }}" is produced automatically?
+    jinja: '{% set label = [''A'',''B'']|choice %}
+
+      Which of the following translations of "{{ orig }}" is produced automatically?
 
-      {{ "A" }}) {{ mt }}
+      {{ "A" }}) {% if label==''A'' %}{{ mt }}{% else %}{{ ref }}{% endif %}
 
-      {{ "B" }}) {{ ref }}
+      {{ "B" }}) {% if label==''A'' %}{{ ref }}{% else %}{{ mt }}{% endif %}
 
-      |||{{ ["A", "B"][0] }}'
+      |||{{ label }}'
     metadata: !TemplateMetadata
       choices_in_prompt: true
       metrics:
       - Accuracy
       original_task: false
-    name: discriminate-mt-ref-A
-    reference: Identify-MT-output, answer=A
+    name: Which is automatic?
+    reference: Identify MT output between MT and ref
   b61c81ec-29eb-47f8-a1c6-561264ac04f3: !Template
     answer_choices: null
     id: b61c81ec-29eb-47f8-a1c6-561264ac04f3
@@ -182,25 +186,9 @@ templates:
       metrics:
       - BLEU
       original_task: true
-    name: MT_5-context_same-lang-mt_both-directions
-    reference: Up to 5 previous sentences (same language, MT), both language directions
-  cfed1937-b4a5-4100-b489-3d8d9cd84423: !Template
-    answer_choices: A ||| B
-    id: cfed1937-b4a5-4100-b489-3d8d9cd84423
-    jinja: 'Which of the following translations of "{{ orig }}" is produced automatically?
-
-      {{ "A" }}) {{ ref }}
-
-      {{ "B" }}) {{ mt }}
-
-      |||{{ ["A", "B"][1] }}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      metrics:
-      - Accuracy
-      original_task: false
-    name: discriminate-mt-ref-B
-    reference: Identify-MT-output, answer=B
+    name: MT with dialogue context (5 same context mt)
+    reference: MT with dialogue context (up to 5 previous sentences, same language
+      MT)
   e9f5dfda-42b5-4698-8e8d-9fc9924e2e29: !Template
     answer_choices: null
     id: e9f5dfda-42b5-4698-8e8d-9fc9924e2e29
@@ -226,8 +214,9 @@ templates:
       metrics:
       - BLEU
       original_task: true
-    name: MT-1-context-complete-translation-orig-both-directions
-    reference: 1 previous context (orig language) complete translation, both directions
+    name: MT complete translation (1 orig context)
+    reference: MT task, complete the translation, 1 sentence of previous context (orig
+      language)
   eea8f47e-9bf5-4423-980b-58a9635c1f49: !Template
     answer_choices: null
     id: eea8f47e-9bf5-4423-980b-58a9635c1f49
@@ -253,5 +242,5 @@ templates:
       metrics:
       - BLEU
       original_task: true
-    name: MT_1-context-analogy_infer-lang_both-directions
-    reference: MT task, 1 previous context (orig language)
+    name: MT analogy, infer lang (1 same context)
+    reference: MT task, translation by analogy (same language), infer target language