From e34db966180c3aaf9c4debcd708823522ed552cd Mon Sep 17 00:00:00 2001
From: Eric Smith <EricMichaelSmith@users.noreply.github.com>
Date: Tue, 11 May 2021 09:02:24 -0400
Subject: [PATCH 1/5] Bump Mephisto version for crowdsourcing checks

Should fix sources of flakiness for crowdsourcing CI tests
---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 5ab983aa084..c0f5496b8ab 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -167,7 +167,7 @@ commands:
             command: |
               cd ..
               git clone git@github.com:facebookresearch/Mephisto.git Mephisto
-              cd Mephisto; git checkout v0.3.2 -b stable
+              cd Mephisto; git checkout v0.3.3 -b stable
               pip install -r requirements.txt
               python setup.py develop
               # `echo` so that ENTER will be pressed at the prompt

From bb9cb408d4989863ff010201bb0b2113f43b9cb7 Mon Sep 17 00:00:00 2001
From: EricMichaelSmith <ems@fb.com>
Date: Wed, 19 May 2021 17:02:38 -0700
Subject: [PATCH 2/5] Partial fix

---
 parlai/crowdsourcing/tasks/acute_eval/conf/base_fast_acute.yaml | 1 +
 parlai/crowdsourcing/tasks/acute_eval/conf/example.yaml         | 1 +
 2 files changed, 2 insertions(+)

diff --git a/parlai/crowdsourcing/tasks/acute_eval/conf/base_fast_acute.yaml b/parlai/crowdsourcing/tasks/acute_eval/conf/base_fast_acute.yaml
index 4c1cbfae45f..f24c07fc6ad 100644
--- a/parlai/crowdsourcing/tasks/acute_eval/conf/base_fast_acute.yaml
+++ b/parlai/crowdsourcing/tasks/acute_eval/conf/base_fast_acute.yaml
@@ -3,6 +3,7 @@ mephisto:
   task:
     allowed_concurrent: 1
     assignment_duration_in_seconds: 600
+    max_num_concurrent_units: 0  # 0 means infinite; set this to a positive integer to limit concurrent HITs and prevent crashes 
     maximum_units_per_worker: 1  # important for replicability and reliability
     task_description: "Evaluate quality of conversations through comparison."
     task_name: acute_eval_${current_time}
diff --git a/parlai/crowdsourcing/tasks/acute_eval/conf/example.yaml b/parlai/crowdsourcing/tasks/acute_eval/conf/example.yaml
index b27480f541b..a0e1a722768 100644
--- a/parlai/crowdsourcing/tasks/acute_eval/conf/example.yaml
+++ b/parlai/crowdsourcing/tasks/acute_eval/conf/example.yaml
@@ -17,6 +17,7 @@ mephisto:
   task:
     allowed_concurrent: 1
     assignment_duration_in_seconds: 600
+    max_num_concurrent_units: 0  # 0 means infinite; set this to a positive integer to limit concurrent HITs and prevent crashes 
     maximum_units_per_worker: 1  # important for replicability and reliability
     task_description: "Evaluate quality of conversations through comparison."
     task_name: acute_eval_${current_time}

From 379348091c7a5374fe1acba04515a73f2548c0ba Mon Sep 17 00:00:00 2001
From: EricMichaelSmith <ems@fb.com>
Date: Wed, 19 May 2021 20:08:55 -0400
Subject: [PATCH 3/5] Reference new argument

---
 docs/source/tutorial_crowdsourcing.md                           | 2 +-
 parlai/crowdsourcing/tasks/model_chat/conf/example.yaml         | 1 +
 .../crowdsourcing/tasks/model_chat/conf/example_image_chat.yaml | 1 +
 parlai/crowdsourcing/tasks/qa_data_collection/conf/example.yaml | 1 +
 .../tasks/turn_annotations_static/conf/example.yaml             | 1 +
 .../turn_annotations_static/conf/example_in_flight_qa.yaml      | 1 +
 6 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/source/tutorial_crowdsourcing.md b/docs/source/tutorial_crowdsourcing.md
index aa22db2fcc0..b5d4d1f01f9 100644
--- a/docs/source/tutorial_crowdsourcing.md
+++ b/docs/source/tutorial_crowdsourcing.md
@@ -210,7 +210,7 @@ Mephisto MTurk Tips and Tricks
 
 ### Preventing and Handling Crashes
 
--   The `max_num_concurrent_units` argument when initializing [`TaskLauncher`](https://github.com/facebookresearch/Mephisto/blob/master/mephisto/operations/task_launcher.py) controls how many people can work on your task at any given time: set this sufficiently low for your task. Leaving this too high might cause your Heroku server to run into issues depending on how many messages per second it's trying to
+-   The `mephisto.task.max_num_concurrent_units` argument controls how many people can work on your task at any given time: set this sufficiently low for your task. Leaving this too high might cause your Heroku server to run into issues depending on how many messages per second it's trying to
     process, and on how much data is being sent in those messages (such
     as picture or video data).
 -   If you're running a model on your local machine, try to share the
diff --git a/parlai/crowdsourcing/tasks/model_chat/conf/example.yaml b/parlai/crowdsourcing/tasks/model_chat/conf/example.yaml
index e06e2d79b9f..635b8e52c20 100644
--- a/parlai/crowdsourcing/tasks/model_chat/conf/example.yaml
+++ b/parlai/crowdsourcing/tasks/model_chat/conf/example.yaml
@@ -15,6 +15,7 @@ mephisto:
     conversations_needed_string: "blender_90M:10"
   task:
     allowed_concurrent: 1
+    max_num_concurrent_units: 0  # 0 means infinite; set this to a positive integer to limit concurrent HITs and prevent crashes
     maximum_units_per_worker: 3
     task_description: |
       <b><h4>Task Description</h4></b>
diff --git a/parlai/crowdsourcing/tasks/model_chat/conf/example_image_chat.yaml b/parlai/crowdsourcing/tasks/model_chat/conf/example_image_chat.yaml
index d4205e46309..eadeeb086d6 100644
--- a/parlai/crowdsourcing/tasks/model_chat/conf/example_image_chat.yaml
+++ b/parlai/crowdsourcing/tasks/model_chat/conf/example_image_chat.yaml
@@ -11,6 +11,7 @@ mephisto:
     num_conversations: 10
   task:
     allowed_concurrent: 1
+    max_num_concurrent_units: 0  # 0 means infinite; set this to a positive integer to limit concurrent HITs and prevent crashes
     maximum_units_per_worker: 3
     task_description: |
       <b><h4>Task Description</h4></b>
diff --git a/parlai/crowdsourcing/tasks/qa_data_collection/conf/example.yaml b/parlai/crowdsourcing/tasks/qa_data_collection/conf/example.yaml
index 1ffdced8f10..d034f1ac60c 100644
--- a/parlai/crowdsourcing/tasks/qa_data_collection/conf/example.yaml
+++ b/parlai/crowdsourcing/tasks/qa_data_collection/conf/example.yaml
@@ -6,6 +6,7 @@ mephisto:
     custom_source_bundle: ${task_dir}/webapp/build/bundle.js
     num_conversations: 1
   task:
+    max_num_concurrent_units: 0  # 0 means infinite; set this to a positive integer to limit concurrent HITs and prevent crashes
     task_name: parlai-qa-example
     task_title: "Test ParlAI QA Data Collection Task"
     task_description: >
diff --git a/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example.yaml b/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example.yaml
index 9f3ee309b57..636bc2fb2a0 100644
--- a/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example.yaml
+++ b/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example.yaml
@@ -11,6 +11,7 @@ mephisto:
     response_field: False
   task:
     allowed_concurrent: 1
+    max_num_concurrent_units: 0  # 0 means infinite; set this to a positive integer to limit concurrent HITs and prevent crashes
     maximum_units_per_worker: 5
     task_description: |
       <b><h4>Task Description</h4></b>
diff --git a/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example_in_flight_qa.yaml b/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example_in_flight_qa.yaml
index 2da7ac71310..f51c45d9257 100644
--- a/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example_in_flight_qa.yaml
+++ b/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example_in_flight_qa.yaml
@@ -9,6 +9,7 @@ mephisto:
     onboarding_qualification: turn_annotations_static
   task:
     allowed_concurrent: 1
+    max_num_concurrent_units: 0  # 0 means infinite; set this to a positive integer to limit concurrent HITs and prevent crashes
     maximum_units_per_worker: 5
     task_description: |
       <b><h4>Task Description</h4></b>

From 64d9bc6171ebe4287ef7d79971291718fb0d71bd Mon Sep 17 00:00:00 2001
From: EricMichaelSmith <ems@fb.com>
Date: Fri, 21 May 2021 18:27:57 -0400
Subject: [PATCH 4/5] Add in more missing flags

---
 parlai/crowdsourcing/tasks/model_chat/conf/example.yaml        | 1 +
 .../tasks/model_chat/conf/example_image_chat.yaml              | 1 +
 .../crowdsourcing/tasks/qa_data_collection/conf/example.yaml   | 3 +++
 .../tasks/turn_annotations_static/conf/example.yaml            | 1 +
 .../turn_annotations_static/conf/example_in_flight_qa.yaml     | 1 +
 5 files changed, 7 insertions(+)

diff --git a/parlai/crowdsourcing/tasks/model_chat/conf/example.yaml b/parlai/crowdsourcing/tasks/model_chat/conf/example.yaml
index 635b8e52c20..6b7bced27b7 100644
--- a/parlai/crowdsourcing/tasks/model_chat/conf/example.yaml
+++ b/parlai/crowdsourcing/tasks/model_chat/conf/example.yaml
@@ -15,6 +15,7 @@ mephisto:
     conversations_needed_string: "blender_90M:10"
   task:
     allowed_concurrent: 1
+    assignment_duration_in_seconds: 600
     max_num_concurrent_units: 0  # 0 means infinite; set this to a positive integer to limit concurrent HITs and prevent crashes
     maximum_units_per_worker: 3
     task_description: |
diff --git a/parlai/crowdsourcing/tasks/model_chat/conf/example_image_chat.yaml b/parlai/crowdsourcing/tasks/model_chat/conf/example_image_chat.yaml
index eadeeb086d6..4f5344af510 100644
--- a/parlai/crowdsourcing/tasks/model_chat/conf/example_image_chat.yaml
+++ b/parlai/crowdsourcing/tasks/model_chat/conf/example_image_chat.yaml
@@ -11,6 +11,7 @@ mephisto:
     num_conversations: 10
   task:
     allowed_concurrent: 1
+    assignment_duration_in_seconds: 600
     max_num_concurrent_units: 0  # 0 means infinite; set this to a positive integer to limit concurrent HITs and prevent crashes
     maximum_units_per_worker: 3
     task_description: |
diff --git a/parlai/crowdsourcing/tasks/qa_data_collection/conf/example.yaml b/parlai/crowdsourcing/tasks/qa_data_collection/conf/example.yaml
index d034f1ac60c..5f64075e906 100644
--- a/parlai/crowdsourcing/tasks/qa_data_collection/conf/example.yaml
+++ b/parlai/crowdsourcing/tasks/qa_data_collection/conf/example.yaml
@@ -6,7 +6,10 @@ mephisto:
     custom_source_bundle: ${task_dir}/webapp/build/bundle.js
     num_conversations: 1
   task:
+    allowed_concurrent: 1
+    assignment_duration_in_seconds: 600
     max_num_concurrent_units: 0  # 0 means infinite; set this to a positive integer to limit concurrent HITs and prevent crashes
+    maximum_units_per_worker: 3
     task_name: parlai-qa-example
     task_title: "Test ParlAI QA Data Collection Task"
     task_description: >
diff --git a/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example.yaml b/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example.yaml
index 636bc2fb2a0..6df4a4a91f7 100644
--- a/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example.yaml
+++ b/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example.yaml
@@ -11,6 +11,7 @@ mephisto:
     response_field: False
   task:
     allowed_concurrent: 1
+    assignment_duration_in_seconds: 600
     max_num_concurrent_units: 0  # 0 means infinite; set this to a positive integer to limit concurrent HITs and prevent crashes
     maximum_units_per_worker: 5
     task_description: |
diff --git a/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example_in_flight_qa.yaml b/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example_in_flight_qa.yaml
index f51c45d9257..99a33805fce 100644
--- a/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example_in_flight_qa.yaml
+++ b/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example_in_flight_qa.yaml
@@ -9,6 +9,7 @@ mephisto:
     onboarding_qualification: turn_annotations_static
   task:
     allowed_concurrent: 1
+    assignment_duration_in_seconds: 600
     max_num_concurrent_units: 0  # 0 means infinite; set this to a positive integer to limit concurrent HITs and prevent crashes
     maximum_units_per_worker: 5
     task_description: |

From 97de3fe228be71baf9df05cbbc693da65d3c3dc1 Mon Sep 17 00:00:00 2001
From: EricMichaelSmith <ems@fb.com>
Date: Mon, 24 May 2021 10:03:02 -0400
Subject: [PATCH 5/5] Fixes

---
 .../tasks/qa_data_collection/test_qa_data_collection.py        | 2 +-
 tests/crowdsourcing/tasks/test_chat_demo.py                    | 3 +++
 .../turn_annotations_static/test_turn_annotations_static.py    | 2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/tests/crowdsourcing/tasks/qa_data_collection/test_qa_data_collection.py b/tests/crowdsourcing/tasks/qa_data_collection/test_qa_data_collection.py
index e4e51fb9760..665727fb280 100644
--- a/tests/crowdsourcing/tasks/qa_data_collection/test_qa_data_collection.py
+++ b/tests/crowdsourcing/tasks/qa_data_collection/test_qa_data_collection.py
@@ -63,7 +63,7 @@ def test_base_task(self):
             # Set up the config and database
             overrides = [
                 'mephisto.blueprint.num_conversations=1',
-                '+mephisto.task.allowed_concurrent=0',
+                'mephisto.task.allowed_concurrent=0',
                 '+turn_timeout=300',
             ]
             # TODO: remove all of these params once Hydra 1.1 is released with
diff --git a/tests/crowdsourcing/tasks/test_chat_demo.py b/tests/crowdsourcing/tasks/test_chat_demo.py
index 56d58c566c5..f58cebfa443 100644
--- a/tests/crowdsourcing/tasks/test_chat_demo.py
+++ b/tests/crowdsourcing/tasks/test_chat_demo.py
@@ -382,6 +382,9 @@ def test_base_task(self):
                 '+mephisto.blueprint.task_description_file=${task_dir}/task_description.html',
                 '+mephisto.blueprint.num_conversations=1',
                 '+mephisto.task.allowed_concurrent=0',
+                '+mephisto.task.assignment_duration_in_seconds=600',
+                '+mephisto.task.max_num_concurrent_units=0',
+                '+mephisto.task.maximum_units_per_worker=0',
                 '+num_turns=3',
                 '+turn_timeout=300',
             ]
diff --git a/tests/crowdsourcing/tasks/turn_annotations_static/test_turn_annotations_static.py b/tests/crowdsourcing/tasks/turn_annotations_static/test_turn_annotations_static.py
index 75581c2863b..cfdf456e5cf 100644
--- a/tests/crowdsourcing/tasks/turn_annotations_static/test_turn_annotations_static.py
+++ b/tests/crowdsourcing/tasks/turn_annotations_static/test_turn_annotations_static.py
@@ -142,7 +142,7 @@ def _test_turn_annotations_static_task(
                 '+mephisto.blueprint.conversation_count=null',
                 'mephisto.blueprint.onboarding_qualification=null',
                 '+mephisto.blueprint.random_seed=42',
-                '+mephisto.task.assignment_duration_in_seconds=1800',
+                'mephisto.task.assignment_duration_in_seconds=1800',
             ]
             # TODO: remove all of these params once Hydra 1.1 is released with support
             #  for recursive defaults