Fix Crowdsourcing Tests after mephisto hydra upgrade

parlai/crowdsourcing/tasks/acute_eval/fast_eval.py

@@ -49,6 +49,8 @@
 # ACUTE EVAL CONSTANTS #
 ########################
 
+FAST_ACUTE_CONFIG_NAME = "example_fast_acute"
+
 ACUTE_EVAL_TYPES = {
     'human': {
         'question': 'Which speaker sounds more human?',
@@ -554,7 +556,7 @@ def analyze_results(self, args: Optional[str] = None):
         self._print_progress(f'ACUTE results saved to {self.results_path}')
 
 
-defaults = ['_self_', {"conf": "example_fast_acute"}]
+defaults = ['_self_', {"conf": FAST_ACUTE_CONFIG_NAME}]
 
 
 @dataclass

parlai/crowdsourcing/tasks/acute_eval/util.py

@@ -76,16 +76,11 @@ def _get_common_overrides(self, root_dir: str) -> List[str]:
         """
         Return overrides for all subclassed Fast ACUTE test code.
         """
-        # TODO: clean this up when Hydra has support for recursive defaults
+
         return [
-            '+mephisto.blueprint.acute_eval_type=engaging',
             'mephisto.blueprint.block_on_onboarding_fail=False',
-            '+mephisto.blueprint.matchups_per_pair=60',
-            '+mephisto.blueprint.num_self_chats=5',
-            f'+mephisto.blueprint.onboarding_path={self.TASK_DIRECTORY}/task_config/onboarding.json',
-            f'+mephisto.blueprint.root_dir={root_dir}',
-            '+mephisto.blueprint.sufficient_matchups_multiplier=2',
-            '+mephisto.blueprint.task=blended_skill_talk',
+            f'mephisto.blueprint.onboarding_path={self.TASK_DIRECTORY}/task_config/onboarding.json',
+            f'mephisto.blueprint.root_dir={root_dir}',
             'mephisto.task.task_name=acute_eval_test',
         ]
 

parlai/crowdsourcing/utils/tests.py

@@ -61,9 +61,9 @@ def _teardown(self):
 
     def _set_up_config(
         self,
-        blueprint_type: str,
         task_directory: str,
         overrides: Optional[List[str]] = None,
+        config_name: str = "example",
     ):
         """
         Set up the config and database.
@@ -87,24 +87,15 @@ def _set_up_config(
             overrides = []
         with initialize(config_path=relative_config_path):
             self.config = compose(
-                config_name="example",
+                config_name=config_name,
                 overrides=[
-                    f'mephisto.blueprint._blueprint_type={blueprint_type}',
-                    f'++mephisto.blueprint.link_task_source=False',
                     f'mephisto/architect=mock',
                     f'mephisto/provider=mock',
                     f'+task_dir={task_directory}',
                     f'+current_time={int(time.time())}',
                 ]
                 + overrides,
             )
-            # TODO: when Hydra 1.1 is released with support for recursive defaults,
-            #  don't manually specify all missing blueprint args anymore, but
-            #  instead define the blueprint in the defaults list directly.
-            #  Currently, the blueprint can't be set in the defaults list without
-            #  overriding params in the YAML file, as documented at
-            #  https://github.com/facebookresearch/hydra/issues/326 and as fixed in
-            #  https://github.com/facebookresearch/hydra/pull/1044.
 
         self.data_dir = tempfile.mkdtemp()
         self.database_path = os.path.join(self.data_dir, "mephisto.db")

tests/crowdsourcing/tasks/acute_eval/test_acute_eval.py

@@ -16,9 +16,6 @@
 
 try:
 
-    from parlai.crowdsourcing.tasks.acute_eval.acute_eval_blueprint import (
-        BLUEPRINT_TYPE,
-    )
     from parlai.crowdsourcing.tasks.acute_eval.run import TASK_DIRECTORY
     from parlai.crowdsourcing.utils.tests import AbstractOneTurnCrowdsourcingTest
 
@@ -55,11 +52,7 @@ def test_base_task(
 
             # Set up the config, database, operator, and server
             overrides = ['mephisto.blueprint.block_on_onboarding_fail=False']
-            self._set_up_config(
-                blueprint_type=BLUEPRINT_TYPE,
-                task_directory=TASK_DIRECTORY,
-                overrides=overrides,
-            )
+            self._set_up_config(task_directory=TASK_DIRECTORY, overrides=overrides)
             self._set_up_server()
 
             # Check that the agent state is as it should be

tests/crowdsourcing/tasks/acute_eval/test_fast_acute_dataset.py

@@ -16,9 +16,9 @@
 
 try:
 
-    from parlai.crowdsourcing.tasks.acute_eval.fast_eval import FastAcuteExecutor
-    from parlai.crowdsourcing.tasks.acute_eval.fast_acute_blueprint import (
-        FAST_ACUTE_BLUEPRINT_TYPE,
+    from parlai.crowdsourcing.tasks.acute_eval.fast_eval import (
+        FastAcuteExecutor,
+        FAST_ACUTE_CONFIG_NAME,
     )
     from parlai.crowdsourcing.tasks.acute_eval.util import AbstractFastAcuteTest
 
@@ -58,21 +58,14 @@ def setup_teardown(self):
 
             # Set up config
             test_overrides = [
-                f'+mephisto.blueprint.config_path={self.TASK_DIRECTORY}/task_config/model_config_dataset.json',
-                f'+mephisto.blueprint.models=\"{self.MODEL_STRING}\"',
-                '+mephisto.blueprint.model_pairs=""',
-                '+mephisto.blueprint.num_task_data_episodes=500',
-                '+mephisto.blueprint.selfchat_max_turns=6',
+                f'mephisto.blueprint.config_path={self.TASK_DIRECTORY}/task_config/model_config_dataset.json',
+                f'mephisto.blueprint.models=\"{self.MODEL_STRING}\"',
             ]
-            # TODO: clean this up when Hydra has support for recursive defaults
             self._set_up_config(
-                blueprint_type=FAST_ACUTE_BLUEPRINT_TYPE,
                 task_directory=self.TASK_DIRECTORY,
                 overrides=self._get_common_overrides(root_dir) + test_overrides,
+                config_name=FAST_ACUTE_CONFIG_NAME,
             )
-            self.config.mephisto.blueprint.model_pairs = None
-            # TODO: hack to manually set mephisto.blueprint.model_pairs to None. Remove
-            #  when Hydra releases support for recursive defaults
 
             # Run Fast ACUTEs
             runner = FastAcuteExecutor(self.config)

tests/crowdsourcing/tasks/acute_eval/test_fast_acute_logs.py

@@ -18,9 +18,9 @@
 
 try:
 
-    from parlai.crowdsourcing.tasks.acute_eval.fast_eval import FastAcuteExecutor
-    from parlai.crowdsourcing.tasks.acute_eval.fast_acute_blueprint import (
-        FAST_ACUTE_BLUEPRINT_TYPE,
+    from parlai.crowdsourcing.tasks.acute_eval.fast_eval import (
+        FastAcuteExecutor,
+        FAST_ACUTE_CONFIG_NAME,
     )
     from parlai.crowdsourcing.tasks.acute_eval.util import AbstractFastAcuteTest
 
@@ -60,19 +60,14 @@ def setup_teardown(self):
             # Set up config
             assert len(self.MODELS) == 2
             test_overrides = [
-                f'+mephisto.blueprint.config_path={config_path}',
-                '+mephisto.blueprint.models=""',
-                f'+mephisto.blueprint.model_pairs={self.MODELS[0]}:{self.MODELS[1]}',
+                f'mephisto.blueprint.config_path={config_path}',
+                f'mephisto.blueprint.model_pairs={self.MODELS[0]}:{self.MODELS[1]}',
             ]
-            # TODO: clean this up when Hydra has support for recursive defaults
             self._set_up_config(
-                blueprint_type=FAST_ACUTE_BLUEPRINT_TYPE,
                 task_directory=self.TASK_DIRECTORY,
                 overrides=self._get_common_overrides(root_dir) + test_overrides,
+                config_name=FAST_ACUTE_CONFIG_NAME,
             )
-            self.config.mephisto.blueprint.models = None
-            # TODO: hack to manually set mephisto.blueprint.models to None. Remove when
-            #  Hydra releases support for recursive defaults
 
             # Save the config file
             config = {}

tests/crowdsourcing/tasks/acute_eval/test_fast_acute_self_chat.py

@@ -17,9 +17,9 @@
 
 try:
 
-    from parlai.crowdsourcing.tasks.acute_eval.fast_eval import FastAcuteExecutor
-    from parlai.crowdsourcing.tasks.acute_eval.fast_acute_blueprint import (
-        FAST_ACUTE_BLUEPRINT_TYPE,
+    from parlai.crowdsourcing.tasks.acute_eval.fast_eval import (
+        FastAcuteExecutor,
+        FAST_ACUTE_CONFIG_NAME,
     )
     from parlai.crowdsourcing.tasks.acute_eval.util import AbstractFastAcuteTest
 
@@ -52,22 +52,12 @@ def setup_teardown(self):
             outputs = {}
 
             # Set up config
-            test_overrides = [
-                f'+mephisto.blueprint.config_path={self.TASK_DIRECTORY}/task_config/model_config_self_chat.json',
-                f'+mephisto.blueprint.models=\"{self.MODEL_STRING}\"',
-                '+mephisto.blueprint.model_pairs=""',
-                '+mephisto.blueprint.selfchat_max_turns=6',
-                '+mephisto.blueprint.use_existing_self_chat_files=True',
-            ]
-            # TODO: clean this up when Hydra has support for recursive defaults
+            test_overrides = ['mephisto.blueprint.use_existing_self_chat_files=True']
             self._set_up_config(
-                blueprint_type=FAST_ACUTE_BLUEPRINT_TYPE,
                 task_directory=self.TASK_DIRECTORY,
                 overrides=self._get_common_overrides(root_dir) + test_overrides,
+                config_name=FAST_ACUTE_CONFIG_NAME,
             )
-            self.config.mephisto.blueprint.model_pairs = None
-            # TODO: hack to manually set mephisto.blueprint.model_pairs to None. Remove
-            #  when Hydra releases support for recursive defaults
 
             # Run Fast ACUTEs
             runner = FastAcuteExecutor(self.config)

tests/crowdsourcing/tasks/model_chat/test_model_chat.py

@@ -63,8 +63,6 @@
     from parlai.crowdsourcing.tasks.model_chat.run import TASK_DIRECTORY
     from parlai.crowdsourcing.tasks.model_chat.model_chat_blueprint import (
         SharedModelChatTaskState,
-        ModelChatBlueprintArgs,
-        BLUEPRINT_TYPE,
     )
     from parlai.crowdsourcing.tasks.model_chat.utils import AbstractModelChatTest
 
@@ -110,36 +108,12 @@ def test_base_task(self):
 
                 # Set up the config and database
                 num_convos = 10
-                args = ModelChatBlueprintArgs()
                 overrides = [
-                    f'++mephisto.blueprint.{key}={val}'
-                    for key, val in args.__dict__.items()
-                    if key
-                    in [
-                        'max_onboard_time',
-                        'max_resp_time',
-                        'override_opt',
-                        'random_seed',
-                        'world_file',
-                    ]
-                ] + [
-                    'mephisto.blueprint.annotations_config_path=${task_dir}/task_config/annotations_config.json',
                     f'mephisto.blueprint.conversations_needed_string=\"fixed_response:{num_convos:d}\"',
                     f'mephisto.blueprint.chat_data_folder={chat_data_folder}',
-                    '++mephisto.blueprint.left_pane_text_path=${task_dir}/task_config/left_pane_text.html',
-                    '++mephisto.blueprint.max_concurrent_responses=1',
                     f'mephisto.blueprint.model_opt_path={model_opt_path}',
-                    f'+mephisto.blueprint.num_conversations={num_convos:d}',
-                    '++mephisto.blueprint.onboard_task_data_path=${task_dir}/task_config/onboard_task_data.json',
-                    '++mephisto.blueprint.task_description_file=${task_dir}/task_config/task_description.html',
                 ]
-                # TODO: remove all of these params once Hydra 1.1 is released with
-                #  support for recursive defaults
-                self._set_up_config(
-                    blueprint_type=BLUEPRINT_TYPE,
-                    task_directory=TASK_DIRECTORY,
-                    overrides=overrides,
-                )
+                self._set_up_config(task_directory=TASK_DIRECTORY, overrides=overrides)
 
                 # Set up the operator and server
                 shared_state = SharedModelChatTaskState(world_module=world_module)

tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py

@@ -36,15 +36,14 @@
 # No info is sent through the 'text' field when submitting the form
 FORM_TASK_DATA = ({"final_rating": 0},)
 
+MODEL_IMAGE_CHAT_CONFIG_NAME = "example_image_chat"
 
 try:
 
     import parlai.crowdsourcing.tasks.model_chat.worlds_image_chat as world_module
     from parlai.crowdsourcing.tasks.model_chat.run_image_chat import TASK_DIRECTORY
     from parlai.crowdsourcing.tasks.model_chat.model_chat_blueprint import (
         SharedModelImageChatTaskState,
-        ModelImageChatBlueprintArgs,
-        IMAGE_CHAT_BLUEPRINT_TYPE,
     )
     from parlai.crowdsourcing.tasks.model_chat.utils import AbstractModelChatTest
 
@@ -117,36 +116,17 @@ def test_base_task(self):
 
                 # Set up the config and database
                 num_convos = 1
-                args = ModelImageChatBlueprintArgs()
                 overrides = [
-                    f'++mephisto.blueprint.{key}={val}'
-                    for key, val in args.__dict__.items()
-                    if key
-                    in [
-                        'evals_per_image_model_combo',
-                        'max_resp_time',
-                        'override_opt',
-                        'random_seed',
-                        'world_file',
-                    ]
-                ] + [
-                    'mephisto.blueprint.annotations_config_path=""',
                     f'mephisto.blueprint.chat_data_folder={chat_data_folder}',
-                    f'+mephisto.blueprint.image_context_path={image_context_path}',
-                    '++mephisto.blueprint.left_pane_text_path=${task_dir}/task_config/left_pane_text.html',
-                    '++mephisto.blueprint.max_concurrent_responses=1',
-                    'mephisto.blueprint.model_opt_path=${task_dir}/task_config/image_model_opts.yaml',
-                    f'+mephisto.blueprint.num_conversations={num_convos:d}',
-                    f'+mephisto.blueprint.stack_folder={stack_folder}',
-                    '++mephisto.blueprint.task_description_file=${task_dir}/task_config/task_description.html',
-                    'mephisto.blueprint.task_model_parallel=False',
+                    f'mephisto.blueprint.image_context_path={image_context_path}',
+                    f'mephisto.blueprint.num_conversations={num_convos:d}',
+                    f'mephisto.blueprint.stack_folder={stack_folder}',
                 ]
-                # TODO: remove all of these params once Hydra 1.1 is released with
-                #  support for recursive defaults
+
                 self._set_up_config(
-                    blueprint_type=IMAGE_CHAT_BLUEPRINT_TYPE,
                     task_directory=TASK_DIRECTORY,
                     overrides=overrides,
+                    config_name=MODEL_IMAGE_CHAT_CONFIG_NAME,
                 )
 
                 # Set up the operator and server

tests/crowdsourcing/tasks/qa_data_collection/test_qa_data_collection.py

@@ -24,7 +24,6 @@
 
     from mephisto.abstractions.blueprints.parlai_chat.parlai_chat_blueprint import (
         SharedParlAITaskState,
-        BLUEPRINT_TYPE,
     )
 
     from parlai.crowdsourcing.tasks.qa_data_collection.run import TASK_DIRECTORY
@@ -61,18 +60,9 @@ def test_base_task(self):
             build_task(task_directory=TASK_DIRECTORY)
 
             # Set up the config and database
-            overrides = [
-                'mephisto.blueprint.num_conversations=1',
-                'mephisto.task.allowed_concurrent=0',
-                '+turn_timeout=300',
-            ]
-            # TODO: remove all of these params once Hydra 1.1 is released with
-            #  support for recursive defaults
-            self._set_up_config(
-                blueprint_type=BLUEPRINT_TYPE,
-                task_directory=TASK_DIRECTORY,
-                overrides=overrides,
-            )
+            overrides = ['+turn_timeout=300']
+
+            self._set_up_config(task_directory=TASK_DIRECTORY, overrides=overrides)
 
             # Set up the operator and server
             teacher = get_teacher(self.config)

tests/crowdsourcing/tasks/test_chat_demo.py

@@ -347,7 +347,6 @@
     import mephisto
     from mephisto.abstractions.blueprints.parlai_chat.parlai_chat_blueprint import (
         SharedParlAITaskState,
-        BLUEPRINT_TYPE,
     )
 
     from parlai.crowdsourcing.utils.tests import AbstractParlAIChatTest
@@ -381,23 +380,14 @@ def test_base_task(self):
 
             # Set up the config and database
             overrides = [
-                '++mephisto.blueprint.world_file=${task_dir}/demo_worlds.py',
-                '++mephisto.blueprint.task_description_file=${task_dir}/task_description.html',
-                '++mephisto.blueprint.num_conversations=1',
                 '++mephisto.task.allowed_concurrent=0',
                 '++mephisto.task.assignment_duration_in_seconds=600',
                 '++mephisto.task.max_num_concurrent_units=0',
                 '++mephisto.task.maximum_units_per_worker=0',
                 '++num_turns=3',
                 '++turn_timeout=300',
             ]
-            # TODO: remove all of these params once Hydra 1.1 is released with support
-            #  for recursive defaults
-            self._set_up_config(
-                blueprint_type=BLUEPRINT_TYPE,
-                task_directory=TASK_DIRECTORY,
-                overrides=overrides,
-            )
+            self._set_up_config(task_directory=TASK_DIRECTORY, overrides=overrides)
 
             # Set up the operator and server
             world_opt = {