Move defaults to config yaml files for crowdsourcing tasks (#4095)

* move defaults for fast acute * add _self_ and move defaults for acute eval * udpated config overrides in test * move defaults for model chat and update test * move defaults for model image chat and update test * add hydra_configs to config path and update test * move defaults and add self for qa data collection task * move defaults to yaml, add self, and update tests * onboarding qualification arg * Fixed Some Typos (#4100) * Add onboarding to crowdsourcing tests (#4096) * add support for registering and onboarding agent * add a value for onboarding qualification instead of null to test * add parameter assume_onboarding and fix spelling * move defaults for fast acute * add _self_ and move defaults for acute eval * udpated config overrides in test * move defaults for model chat and update test * move defaults for model image chat and update test * add hydra_configs to config path and update test * move defaults and add self for qa data collection task * move defaults to yaml, add self, and update tests * onboarding qualification arg * rebase * rebase more edits and cleanup * linting * placeholder to remove unused import error * move defaults for tasks in crowdsourcing projects * point to new mephisto release for circle ci Co-authored-by: Atharv jairath <54663702+atharvjairath@users.noreply.github.com>
facebookresearch · Oct 25, 2021 · 018a407 · 018a407
1 parent 1bf1c81
commit 018a407
Show file tree

Hide file tree

Showing 26 changed files with 87 additions and 98 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -167,7 +167,7 @@ commands:
             command: |
               cd ..
               git clone git@github.com:facebookresearch/Mephisto.git Mephisto
-              cd Mephisto; git checkout v0.4.0 -b stable
+              cd Mephisto; git checkout v0.4.1 -b stable
               pip install -r requirements.txt
               python setup.py develop
               # `echo` so that ENTER will be pressed at the prompt

diff --git a/parlai/crowdsourcing/projects/multisession_chat/human_eval/hydra_configs/conf/example.yaml b/parlai/crowdsourcing/projects/multisession_chat/human_eval/hydra_configs/conf/example.yaml
@@ -1,4 +1,8 @@
 #@package _global_
+defaults:
+  - /mephisto/blueprint: model_chat_blueprint
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   blueprint:
     annotations_config_path: ${task_dir}/task_config/annotations_config.json

diff --git a/parlai/crowdsourcing/projects/multisession_chat/human_eval/run.py b/parlai/crowdsourcing/projects/multisession_chat/human_eval/run.py
@@ -18,15 +18,12 @@
 import json
 
 
+_ = BLUEPRINT_TYPE
+
 TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
 
 
-defaults = [
-    {'mephisto/blueprint': BLUEPRINT_TYPE},
-    {"mephisto/architect": "local"},
-    {"mephisto/provider": "mock"},
-    {"conf": "example"},
-]
+defaults = ["_self_", {"conf": "example"}]
 
 
 def check_override_opt(args):

diff --git a/parlai/crowdsourcing/projects/wizard_of_internet/hydra_configs/conf/dev.yaml b/parlai/crowdsourcing/projects/wizard_of_internet/hydra_configs/conf/dev.yaml
@@ -1,5 +1,9 @@
 
 #@package _global_
+defaults:
+  - /mephisto/blueprint: wizard_internet_parlaichat_blueprint
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   blueprint:
     onboarding_qualification: wizard-onboarding-dev

diff --git a/parlai/crowdsourcing/projects/wizard_of_internet/run.py b/parlai/crowdsourcing/projects/wizard_of_internet/run.py
@@ -28,14 +28,11 @@
 from mephisto.tools.scripts import load_db_and_process_config
 
 
+_ = WIZARD_INTERNET_PARLAICHAT_BLUEPRINT
+
 TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
 
-defaults = [
-    {'mephisto/blueprint': WIZARD_INTERNET_PARLAICHAT_BLUEPRINT},
-    {'mephisto/architect': 'local'},
-    {'mephisto/provider': 'mock'},
-    {'conf': 'dev'},
-]
+defaults = ['_self_', {"conf": "dev"}]
 
 
 @dataclass

diff --git a/parlai/crowdsourcing/tasks/acute_eval/fast_eval.py b/parlai/crowdsourcing/tasks/acute_eval/fast_eval.py
@@ -43,6 +43,8 @@
 from parlai.utils.strings import normalize_reply
 from parlai.utils.testing import capture_output
 
+_ = FAST_ACUTE_BLUEPRINT_TYPE
+
 ########################
 # ACUTE EVAL CONSTANTS #
 ########################
@@ -547,14 +549,7 @@ def analyze_results(self, args: Optional[str] = None):
         self._print_progress(f'ACUTE results saved to {self.results_path}')
 
 
-defaults = [
-    '_self_',
-    {"mephisto/blueprint": FAST_ACUTE_BLUEPRINT_TYPE},
-    {"mephisto/architect": "local"},
-    {"mephisto/provider": "mock"},
-    'conf/base_fast_acute',
-    {"conf": "example_fast_acute"},
-]
+defaults = ['_self_', {"conf": "example_fast_acute"}]
 
 
 @dataclass

diff --git a/parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/base_fast_acute.yaml b/parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/base_fast_acute.yaml
@@ -1,4 +1,8 @@
 #@package _global_
+defaults:
+  - /mephisto/blueprint: fast_acute
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   task:
     allowed_concurrent: 1

diff --git a/parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/example.yaml b/parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/example.yaml
@@ -1,4 +1,8 @@
 #@package _global_
+defaults:
+  - /mephisto/blueprint: acute_eval
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   blueprint:
     additional_task_description: ""

diff --git a/parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/example_fast_acute.yaml b/parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/example_fast_acute.yaml
@@ -1,4 +1,6 @@
 #@package _global_
+defaults:
+  - base_fast_acute
 mephisto:
   blueprint:
     acute_eval_type: engaging

diff --git a/parlai/crowdsourcing/tasks/acute_eval/run.py b/parlai/crowdsourcing/tasks/acute_eval/run.py
@@ -30,12 +30,9 @@
 
 """
 
-defaults = [
-    {"mephisto/blueprint": BLUEPRINT_TYPE},
-    {"mephisto/architect": "local"},
-    {"mephisto/provider": "mock"},
-    {"conf": "example"},
-]
+_ = BLUEPRINT_TYPE
+
+defaults = ["_self_", {"conf": "example"}]
 
 
 @dataclass

diff --git a/parlai/crowdsourcing/tasks/chat_demo/run.py b/parlai/crowdsourcing/tasks/chat_demo/run.py
@@ -41,9 +41,10 @@ class ScriptConfig(MTurkRunScriptConfigMixin, TestScriptConfig):
 
 register_script_config(name="scriptconfig", module=ScriptConfig)
 relative_task_directory = os.path.relpath(TASK_DIRECTORY, os.path.dirname(__file__))
+config_path = os.path.join(relative_task_directory, 'hydra_configs')
 
 
-@hydra.main(config_path=relative_task_directory, config_name="scriptconfig")
+@hydra.main(config_path=config_path, config_name="scriptconfig")
 def main(cfg: DictConfig) -> None:
     db, cfg = load_db_and_process_config(cfg)
 

diff --git a/parlai/crowdsourcing/tasks/model_chat/hydra_configs/conf/example.yaml b/parlai/crowdsourcing/tasks/model_chat/hydra_configs/conf/example.yaml
@@ -1,4 +1,8 @@
 #@package _global_
+defaults:
+  - /mephisto/blueprint: model_chat_blueprint
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   blueprint:
     annotations_config_path: ${task_dir}/task_config/annotations_config.json

diff --git a/parlai/crowdsourcing/tasks/model_chat/hydra_configs/conf/example_image_chat.yaml b/parlai/crowdsourcing/tasks/model_chat/hydra_configs/conf/example_image_chat.yaml
@@ -1,4 +1,8 @@
 #@package _global_
+defaults:
+  - /mephisto/blueprint: model_image_chat_blueprint
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   blueprint:
     annotations_config_path: ""

diff --git a/parlai/crowdsourcing/tasks/model_chat/run.py b/parlai/crowdsourcing/tasks/model_chat/run.py
@@ -12,7 +12,6 @@
 from mephisto.operations.hydra_config import register_script_config
 from omegaconf import DictConfig
 
-from parlai.crowdsourcing.tasks.model_chat.model_chat_blueprint import BLUEPRINT_TYPE
 from parlai.crowdsourcing.tasks.model_chat.impl import run_task
 from parlai.crowdsourcing.utils.mturk import MTurkRunScriptConfig
 import parlai.crowdsourcing.tasks.model_chat.worlds as world_module
@@ -21,12 +20,7 @@
 TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
 
 
-defaults = [
-    {'mephisto/blueprint': BLUEPRINT_TYPE},
-    {"mephisto/architect": "local"},
-    {"mephisto/provider": "mock"},
-    {"conf": "example"},
-]
+defaults = ["_self_", {"conf": "example"}]
 
 
 @dataclass

diff --git a/parlai/crowdsourcing/tasks/model_chat/run_image_chat.py b/parlai/crowdsourcing/tasks/model_chat/run_image_chat.py
@@ -12,9 +12,6 @@
 from mephisto.operations.hydra_config import register_script_config
 from omegaconf import DictConfig
 
-from parlai.crowdsourcing.tasks.model_chat.model_chat_blueprint import (
-    IMAGE_CHAT_BLUEPRINT_TYPE,
-)
 from parlai.crowdsourcing.tasks.model_chat.impl import run_task
 from parlai.crowdsourcing.utils.mturk import MTurkRunScriptConfig
 import parlai.crowdsourcing.tasks.model_chat.worlds as world_module
@@ -23,12 +20,7 @@
 TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
 
 
-defaults = [
-    {'mephisto/blueprint': IMAGE_CHAT_BLUEPRINT_TYPE},
-    {"mephisto/architect": "local"},
-    {"mephisto/provider": "mock"},
-    {"conf": "example_image_chat"},
-]
+defaults = ["_self_", {"conf": "example_image_chat"}]
 
 
 @dataclass

diff --git a/parlai/crowdsourcing/tasks/qa_data_collection/hydra_configs/conf/example.yaml b/parlai/crowdsourcing/tasks/qa_data_collection/hydra_configs/conf/example.yaml
@@ -1,4 +1,8 @@
 #@package _global_
+defaults:
+  - /mephisto/blueprint: parlai_chat
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   blueprint:
     world_file: ${task_dir}/worlds.py

diff --git a/parlai/crowdsourcing/tasks/qa_data_collection/run.py b/parlai/crowdsourcing/tasks/qa_data_collection/run.py
@@ -12,7 +12,6 @@
 import hydra
 from omegaconf import DictConfig
 from mephisto.abstractions.blueprints.parlai_chat.parlai_chat_blueprint import (
-    BLUEPRINT_TYPE,
     SharedParlAITaskState,
 )
 from mephisto.operations.hydra_config import register_script_config
@@ -26,12 +25,7 @@
 
 TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
 
-defaults = [
-    {"mephisto/blueprint": BLUEPRINT_TYPE},
-    {"mephisto/architect": "local"},
-    {"mephisto/provider": "mock"},
-    {"conf": "example"},
-]
+defaults = ["_self_", {"conf": "example"}]
 
 
 @dataclass

diff --git a/parlai/crowdsourcing/tasks/turn_annotations_static/hydra_configs/conf/example.yaml b/parlai/crowdsourcing/tasks/turn_annotations_static/hydra_configs/conf/example.yaml
@@ -1,4 +1,8 @@
 #@package _global_
+defaults:
+  - /mephisto/blueprint: turn_annotations_static_blueprint
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   blueprint:
     subtasks_per_unit: 6

diff --git a/.../crowdsourcing/tasks/turn_annotations_static/hydra_configs/conf/example_in_flight_qa.yaml b/.../crowdsourcing/tasks/turn_annotations_static/hydra_configs/conf/example_in_flight_qa.yaml
@@ -1,4 +1,8 @@
 #@package _global_
+defaults:
+  - /mephisto/blueprint: turn_annotations_static_inflight_qa_blueprint
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   blueprint:
     subtasks_per_unit: 6

diff --git a/parlai/crowdsourcing/tasks/turn_annotations_static/run.py b/parlai/crowdsourcing/tasks/turn_annotations_static/run.py
@@ -12,9 +12,6 @@
 from mephisto.operations.hydra_config import register_script_config
 from omegaconf import DictConfig
 
-from parlai.crowdsourcing.tasks.turn_annotations_static.turn_annotations_blueprint import (
-    STATIC_BLUEPRINT_TYPE,
-)
 from parlai.crowdsourcing.tasks.turn_annotations_static.util import run_static_task
 from parlai.crowdsourcing.utils.mturk import MTurkRunScriptConfig
 
@@ -26,12 +23,7 @@
 # check the README in parlai/crowdsourcing/
 # python turn_annotations_static/run.py conf=<conf name sans yaml> --config-dir <path to directory with a conf/ folder>
 
-defaults = [
-    {'mephisto/blueprint': STATIC_BLUEPRINT_TYPE},
-    {"mephisto/architect": "local"},
-    {"mephisto/provider": "mock"},
-    {"conf": 'example'},
-]
+defaults = ["_self_", {"conf": 'example'}]
 
 
 @dataclass

diff --git a/parlai/crowdsourcing/tasks/turn_annotations_static/run_in_flight_qa.py b/parlai/crowdsourcing/tasks/turn_annotations_static/run_in_flight_qa.py
@@ -12,9 +12,6 @@
 from mephisto.operations.hydra_config import register_script_config
 from omegaconf import DictConfig
 
-from parlai.crowdsourcing.tasks.turn_annotations_static.turn_annotations_blueprint import (
-    STATIC_IN_FLIGHT_QA_BLUEPRINT_TYPE,
-)
 from parlai.crowdsourcing.tasks.turn_annotations_static.util import run_static_task
 from parlai.crowdsourcing.utils.mturk import MTurkRunScriptConfig
 
@@ -26,12 +23,7 @@
 TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
 
 
-defaults = [
-    {'mephisto/blueprint': STATIC_IN_FLIGHT_QA_BLUEPRINT_TYPE},
-    {"mephisto/architect": "local"},
-    {"mephisto/provider": "mock"},
-    {"conf": "example_in_flight_qa"},
-]
+defaults = ["_self_", {"conf": "example_in_flight_qa"}]
 
 
 @dataclass

diff --git a/parlai/crowdsourcing/utils/tests.py b/parlai/crowdsourcing/utils/tests.py
@@ -89,10 +89,10 @@ def _set_up_config(
             self.config = compose(
                 config_name="example",
                 overrides=[
-                    f'+mephisto.blueprint._blueprint_type={blueprint_type}',
-                    f'+mephisto.blueprint.link_task_source=False',
-                    f'+mephisto/architect=mock',
-                    f'+mephisto/provider=mock',
+                    f'mephisto.blueprint._blueprint_type={blueprint_type}',
+                    f'++mephisto.blueprint.link_task_source=False',
+                    f'mephisto/architect=mock',
+                    f'mephisto/provider=mock',
                     f'+task_dir={task_directory}',
                     f'+current_time={int(time.time())}',
                 ]
@@ -136,8 +136,8 @@ def _register_mock_agents(
         self, num_agents: int = 1, assume_onboarding: bool = False
     ) -> List[str]:
         """
-        Register mock agents for testing and onboard them if needed, 
-        taking the place of crowdsourcing workers.
+        Register mock agents for testing and onboard them if needed, taking the place of
+        crowdsourcing workers.
 
         Specify the number of agents to register. Return the agents' IDs after creation.
         """

diff --git a/tests/crowdsourcing/tasks/model_chat/test_model_chat.py b/tests/crowdsourcing/tasks/model_chat/test_model_chat.py
@@ -112,7 +112,7 @@ def test_base_task(self):
                 num_convos = 10
                 args = ModelChatBlueprintArgs()
                 overrides = [
-                    f'+mephisto.blueprint.{key}={val}'
+                    f'++mephisto.blueprint.{key}={val}'
                     for key, val in args.__dict__.items()
                     if key
                     in [
@@ -126,12 +126,12 @@ def test_base_task(self):
                     'mephisto.blueprint.annotations_config_path=${task_dir}/task_config/annotations_config.json',
                     f'mephisto.blueprint.conversations_needed_string=\"fixed_response:{num_convos:d}\"',
                     f'mephisto.blueprint.chat_data_folder={chat_data_folder}',
-                    '+mephisto.blueprint.left_pane_text_path=${task_dir}/task_config/left_pane_text.html',
-                    '+mephisto.blueprint.max_concurrent_responses=1',
+                    '++mephisto.blueprint.left_pane_text_path=${task_dir}/task_config/left_pane_text.html',
+                    '++mephisto.blueprint.max_concurrent_responses=1',
                     f'mephisto.blueprint.model_opt_path={model_opt_path}',
                     f'+mephisto.blueprint.num_conversations={num_convos:d}',
-                    '+mephisto.blueprint.onboard_task_data_path=${task_dir}/task_config/onboard_task_data.json',
-                    '+mephisto.blueprint.task_description_file=${task_dir}/task_config/task_description.html',
+                    '++mephisto.blueprint.onboard_task_data_path=${task_dir}/task_config/onboard_task_data.json',
+                    '++mephisto.blueprint.task_description_file=${task_dir}/task_config/task_description.html',
                 ]
                 # TODO: remove all of these params once Hydra 1.1 is released with
                 #  support for recursive defaults

diff --git a/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py b/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py
@@ -119,7 +119,7 @@ def test_base_task(self):
                 num_convos = 1
                 args = ModelImageChatBlueprintArgs()
                 overrides = [
-                    f'+mephisto.blueprint.{key}={val}'
+                    f'++mephisto.blueprint.{key}={val}'
                     for key, val in args.__dict__.items()
                     if key
                     in [
@@ -133,12 +133,12 @@ def test_base_task(self):
                     'mephisto.blueprint.annotations_config_path=""',
                     f'mephisto.blueprint.chat_data_folder={chat_data_folder}',
                     f'+mephisto.blueprint.image_context_path={image_context_path}',
-                    '+mephisto.blueprint.left_pane_text_path=${task_dir}/task_config/left_pane_text.html',
-                    '+mephisto.blueprint.max_concurrent_responses=1',
+                    '++mephisto.blueprint.left_pane_text_path=${task_dir}/task_config/left_pane_text.html',
+                    '++mephisto.blueprint.max_concurrent_responses=1',
                     'mephisto.blueprint.model_opt_path=${task_dir}/task_config/image_model_opts.yaml',
                     f'+mephisto.blueprint.num_conversations={num_convos:d}',
                     f'+mephisto.blueprint.stack_folder={stack_folder}',
-                    '+mephisto.blueprint.task_description_file=${task_dir}/task_config/task_description.html',
+                    '++mephisto.blueprint.task_description_file=${task_dir}/task_config/task_description.html',
                     'mephisto.blueprint.task_model_parallel=False',
                 ]
                 # TODO: remove all of these params once Hydra 1.1 is released with