From c5f43b39eb32641faf876708065fa99b437556e4 Mon Sep 17 00:00:00 2001 From: meganung Date: Tue, 7 Dec 2021 20:40:48 -0800 Subject: [PATCH 1/5] remove unnecessary overrides that are already defined, remove blueprint and point to yaml --- .../tasks/acute_eval/fast_eval.py | 4 ++- parlai/crowdsourcing/tasks/acute_eval/util.py | 11 ++----- parlai/crowdsourcing/utils/tests.py | 6 ++-- .../tasks/acute_eval/test_acute_eval.py | 9 +---- .../acute_eval/test_fast_acute_dataset.py | 19 ++++------- .../tasks/acute_eval/test_fast_acute_logs.py | 17 ++++------ .../acute_eval/test_fast_acute_self_chat.py | 20 +++-------- .../tasks/model_chat/test_model_chat.py | 10 +----- .../tasks/model_chat/test_model_image_chat.py | 17 ++++------ .../test_qa_data_collection.py | 16 ++------- tests/crowdsourcing/tasks/test_chat_demo.py | 11 +------ .../test_turn_annotations_static.py | 33 ++++--------------- 12 files changed, 44 insertions(+), 129 deletions(-) diff --git a/parlai/crowdsourcing/tasks/acute_eval/fast_eval.py b/parlai/crowdsourcing/tasks/acute_eval/fast_eval.py index 7d4a4c94c16..583d06fc890 100644 --- a/parlai/crowdsourcing/tasks/acute_eval/fast_eval.py +++ b/parlai/crowdsourcing/tasks/acute_eval/fast_eval.py @@ -49,6 +49,8 @@ # ACUTE EVAL CONSTANTS # ######################## +FAST_ACUTE_CONFIG_NAME = "example_fast_acute" + ACUTE_EVAL_TYPES = { 'human': { 'question': 'Which speaker sounds more human?', @@ -554,7 +556,7 @@ def analyze_results(self, args: Optional[str] = None): self._print_progress(f'ACUTE results saved to {self.results_path}') -defaults = ['_self_', {"conf": "example_fast_acute"}] +defaults = ['_self_', {"conf": FAST_ACUTE_CONFIG_NAME}] @dataclass diff --git a/parlai/crowdsourcing/tasks/acute_eval/util.py b/parlai/crowdsourcing/tasks/acute_eval/util.py index eec5206c569..440f22bf020 100644 --- a/parlai/crowdsourcing/tasks/acute_eval/util.py +++ b/parlai/crowdsourcing/tasks/acute_eval/util.py @@ -76,16 +76,11 @@ def _get_common_overrides(self, root_dir: str) -> List[str]: """ Return overrides for all subclassed Fast ACUTE test code. """ - # TODO: clean this up when Hydra has support for recursive defaults + return [ - '+mephisto.blueprint.acute_eval_type=engaging', 'mephisto.blueprint.block_on_onboarding_fail=False', - '+mephisto.blueprint.matchups_per_pair=60', - '+mephisto.blueprint.num_self_chats=5', - f'+mephisto.blueprint.onboarding_path={self.TASK_DIRECTORY}/task_config/onboarding.json', - f'+mephisto.blueprint.root_dir={root_dir}', - '+mephisto.blueprint.sufficient_matchups_multiplier=2', - '+mephisto.blueprint.task=blended_skill_talk', + f'mephisto.blueprint.onboarding_path={self.TASK_DIRECTORY}/task_config/onboarding.json', + f'mephisto.blueprint.root_dir={root_dir}', 'mephisto.task.task_name=acute_eval_test', ] diff --git a/parlai/crowdsourcing/utils/tests.py b/parlai/crowdsourcing/utils/tests.py index f4d8e085e70..2836d0597aa 100644 --- a/parlai/crowdsourcing/utils/tests.py +++ b/parlai/crowdsourcing/utils/tests.py @@ -61,9 +61,9 @@ def _teardown(self): def _set_up_config( self, - blueprint_type: str, task_directory: str, overrides: Optional[List[str]] = None, + config_name: str = "example", ): """ Set up the config and database. @@ -87,10 +87,8 @@ def _set_up_config( overrides = [] with initialize(config_path=relative_config_path): self.config = compose( - config_name="example", + config_name=config_name, overrides=[ - f'mephisto.blueprint._blueprint_type={blueprint_type}', - f'++mephisto.blueprint.link_task_source=False', f'mephisto/architect=mock', f'mephisto/provider=mock', f'+task_dir={task_directory}', diff --git a/tests/crowdsourcing/tasks/acute_eval/test_acute_eval.py b/tests/crowdsourcing/tasks/acute_eval/test_acute_eval.py index 90e1800bcc9..2257fb51e49 100644 --- a/tests/crowdsourcing/tasks/acute_eval/test_acute_eval.py +++ b/tests/crowdsourcing/tasks/acute_eval/test_acute_eval.py @@ -16,9 +16,6 @@ try: - from parlai.crowdsourcing.tasks.acute_eval.acute_eval_blueprint import ( - BLUEPRINT_TYPE, - ) from parlai.crowdsourcing.tasks.acute_eval.run import TASK_DIRECTORY from parlai.crowdsourcing.utils.tests import AbstractOneTurnCrowdsourcingTest @@ -55,11 +52,7 @@ def test_base_task( # Set up the config, database, operator, and server overrides = ['mephisto.blueprint.block_on_onboarding_fail=False'] - self._set_up_config( - blueprint_type=BLUEPRINT_TYPE, - task_directory=TASK_DIRECTORY, - overrides=overrides, - ) + self._set_up_config(task_directory=TASK_DIRECTORY, overrides=overrides) self._set_up_server() # Check that the agent state is as it should be diff --git a/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_dataset.py b/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_dataset.py index 802294d5fe4..74ef19990b2 100644 --- a/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_dataset.py +++ b/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_dataset.py @@ -16,9 +16,9 @@ try: - from parlai.crowdsourcing.tasks.acute_eval.fast_eval import FastAcuteExecutor - from parlai.crowdsourcing.tasks.acute_eval.fast_acute_blueprint import ( - FAST_ACUTE_BLUEPRINT_TYPE, + from parlai.crowdsourcing.tasks.acute_eval.fast_eval import ( + FastAcuteExecutor, + FAST_ACUTE_CONFIG_NAME, ) from parlai.crowdsourcing.tasks.acute_eval.util import AbstractFastAcuteTest @@ -58,21 +58,14 @@ def setup_teardown(self): # Set up config test_overrides = [ - f'+mephisto.blueprint.config_path={self.TASK_DIRECTORY}/task_config/model_config_dataset.json', - f'+mephisto.blueprint.models=\"{self.MODEL_STRING}\"', - '+mephisto.blueprint.model_pairs=""', - '+mephisto.blueprint.num_task_data_episodes=500', - '+mephisto.blueprint.selfchat_max_turns=6', + f'mephisto.blueprint.config_path={self.TASK_DIRECTORY}/task_config/model_config_dataset.json', + f'mephisto.blueprint.models=\"{self.MODEL_STRING}\"', ] - # TODO: clean this up when Hydra has support for recursive defaults self._set_up_config( - blueprint_type=FAST_ACUTE_BLUEPRINT_TYPE, task_directory=self.TASK_DIRECTORY, overrides=self._get_common_overrides(root_dir) + test_overrides, + config_name=FAST_ACUTE_CONFIG_NAME, ) - self.config.mephisto.blueprint.model_pairs = None - # TODO: hack to manually set mephisto.blueprint.model_pairs to None. Remove - # when Hydra releases support for recursive defaults # Run Fast ACUTEs runner = FastAcuteExecutor(self.config) diff --git a/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_logs.py b/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_logs.py index 718ebdc6943..85f3bfac5b1 100644 --- a/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_logs.py +++ b/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_logs.py @@ -18,9 +18,9 @@ try: - from parlai.crowdsourcing.tasks.acute_eval.fast_eval import FastAcuteExecutor - from parlai.crowdsourcing.tasks.acute_eval.fast_acute_blueprint import ( - FAST_ACUTE_BLUEPRINT_TYPE, + from parlai.crowdsourcing.tasks.acute_eval.fast_eval import ( + FastAcuteExecutor, + FAST_ACUTE_CONFIG_NAME, ) from parlai.crowdsourcing.tasks.acute_eval.util import AbstractFastAcuteTest @@ -60,19 +60,14 @@ def setup_teardown(self): # Set up config assert len(self.MODELS) == 2 test_overrides = [ - f'+mephisto.blueprint.config_path={config_path}', - '+mephisto.blueprint.models=""', - f'+mephisto.blueprint.model_pairs={self.MODELS[0]}:{self.MODELS[1]}', + f'mephisto.blueprint.config_path={config_path}', + f'mephisto.blueprint.model_pairs={self.MODELS[0]}:{self.MODELS[1]}', ] - # TODO: clean this up when Hydra has support for recursive defaults self._set_up_config( - blueprint_type=FAST_ACUTE_BLUEPRINT_TYPE, task_directory=self.TASK_DIRECTORY, overrides=self._get_common_overrides(root_dir) + test_overrides, + config_name=FAST_ACUTE_CONFIG_NAME, ) - self.config.mephisto.blueprint.models = None - # TODO: hack to manually set mephisto.blueprint.models to None. Remove when - # Hydra releases support for recursive defaults # Save the config file config = {} diff --git a/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_self_chat.py b/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_self_chat.py index 899a7bdd089..bae3ded63b5 100644 --- a/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_self_chat.py +++ b/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_self_chat.py @@ -17,9 +17,9 @@ try: - from parlai.crowdsourcing.tasks.acute_eval.fast_eval import FastAcuteExecutor - from parlai.crowdsourcing.tasks.acute_eval.fast_acute_blueprint import ( - FAST_ACUTE_BLUEPRINT_TYPE, + from parlai.crowdsourcing.tasks.acute_eval.fast_eval import ( + FastAcuteExecutor, + FAST_ACUTE_CONFIG_NAME, ) from parlai.crowdsourcing.tasks.acute_eval.util import AbstractFastAcuteTest @@ -52,22 +52,12 @@ def setup_teardown(self): outputs = {} # Set up config - test_overrides = [ - f'+mephisto.blueprint.config_path={self.TASK_DIRECTORY}/task_config/model_config_self_chat.json', - f'+mephisto.blueprint.models=\"{self.MODEL_STRING}\"', - '+mephisto.blueprint.model_pairs=""', - '+mephisto.blueprint.selfchat_max_turns=6', - '+mephisto.blueprint.use_existing_self_chat_files=True', - ] - # TODO: clean this up when Hydra has support for recursive defaults + test_overrides = ['mephisto.blueprint.use_existing_self_chat_files=True'] self._set_up_config( - blueprint_type=FAST_ACUTE_BLUEPRINT_TYPE, task_directory=self.TASK_DIRECTORY, overrides=self._get_common_overrides(root_dir) + test_overrides, + config_name=FAST_ACUTE_CONFIG_NAME, ) - self.config.mephisto.blueprint.model_pairs = None - # TODO: hack to manually set mephisto.blueprint.model_pairs to None. Remove - # when Hydra releases support for recursive defaults # Run Fast ACUTEs runner = FastAcuteExecutor(self.config) diff --git a/tests/crowdsourcing/tasks/model_chat/test_model_chat.py b/tests/crowdsourcing/tasks/model_chat/test_model_chat.py index 40f4fcb6cdc..75f252de8d6 100644 --- a/tests/crowdsourcing/tasks/model_chat/test_model_chat.py +++ b/tests/crowdsourcing/tasks/model_chat/test_model_chat.py @@ -126,20 +126,12 @@ def test_base_task(self): 'mephisto.blueprint.annotations_config_path=${task_dir}/task_config/annotations_config.json', f'mephisto.blueprint.conversations_needed_string=\"fixed_response:{num_convos:d}\"', f'mephisto.blueprint.chat_data_folder={chat_data_folder}', - '++mephisto.blueprint.left_pane_text_path=${task_dir}/task_config/left_pane_text.html', - '++mephisto.blueprint.max_concurrent_responses=1', f'mephisto.blueprint.model_opt_path={model_opt_path}', f'+mephisto.blueprint.num_conversations={num_convos:d}', - '++mephisto.blueprint.onboard_task_data_path=${task_dir}/task_config/onboard_task_data.json', - '++mephisto.blueprint.task_description_file=${task_dir}/task_config/task_description.html', ] # TODO: remove all of these params once Hydra 1.1 is released with # support for recursive defaults - self._set_up_config( - blueprint_type=BLUEPRINT_TYPE, - task_directory=TASK_DIRECTORY, - overrides=overrides, - ) + self._set_up_config(task_directory=TASK_DIRECTORY, overrides=overrides) # Set up the operator and server shared_state = SharedModelChatTaskState(world_module=world_module) diff --git a/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py b/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py index b60787c7cb6..a1d4e6570e7 100644 --- a/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py +++ b/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py @@ -36,6 +36,7 @@ # No info is sent through the 'text' field when submitting the form FORM_TASK_DATA = ({"final_rating": 0},) +MODEL_IMAGE_CHAT_CONFIG_NAME = "example_image_chat" try: @@ -130,23 +131,17 @@ def test_base_task(self): 'world_file', ] ] + [ - 'mephisto.blueprint.annotations_config_path=""', f'mephisto.blueprint.chat_data_folder={chat_data_folder}', - f'+mephisto.blueprint.image_context_path={image_context_path}', - '++mephisto.blueprint.left_pane_text_path=${task_dir}/task_config/left_pane_text.html', - '++mephisto.blueprint.max_concurrent_responses=1', - 'mephisto.blueprint.model_opt_path=${task_dir}/task_config/image_model_opts.yaml', - f'+mephisto.blueprint.num_conversations={num_convos:d}', - f'+mephisto.blueprint.stack_folder={stack_folder}', - '++mephisto.blueprint.task_description_file=${task_dir}/task_config/task_description.html', + f'mephisto.blueprint.image_context_path={image_context_path}', + f'mephisto.blueprint.num_conversations={num_convos:d}', + f'mephisto.blueprint.stack_folder={stack_folder}', 'mephisto.blueprint.task_model_parallel=False', ] - # TODO: remove all of these params once Hydra 1.1 is released with - # support for recursive defaults + self._set_up_config( - blueprint_type=IMAGE_CHAT_BLUEPRINT_TYPE, task_directory=TASK_DIRECTORY, overrides=overrides, + config_name=MODEL_IMAGE_CHAT_CONFIG_NAME, ) # Set up the operator and server diff --git a/tests/crowdsourcing/tasks/qa_data_collection/test_qa_data_collection.py b/tests/crowdsourcing/tasks/qa_data_collection/test_qa_data_collection.py index 665727fb280..49bd70269df 100644 --- a/tests/crowdsourcing/tasks/qa_data_collection/test_qa_data_collection.py +++ b/tests/crowdsourcing/tasks/qa_data_collection/test_qa_data_collection.py @@ -24,7 +24,6 @@ from mephisto.abstractions.blueprints.parlai_chat.parlai_chat_blueprint import ( SharedParlAITaskState, - BLUEPRINT_TYPE, ) from parlai.crowdsourcing.tasks.qa_data_collection.run import TASK_DIRECTORY @@ -61,18 +60,9 @@ def test_base_task(self): build_task(task_directory=TASK_DIRECTORY) # Set up the config and database - overrides = [ - 'mephisto.blueprint.num_conversations=1', - 'mephisto.task.allowed_concurrent=0', - '+turn_timeout=300', - ] - # TODO: remove all of these params once Hydra 1.1 is released with - # support for recursive defaults - self._set_up_config( - blueprint_type=BLUEPRINT_TYPE, - task_directory=TASK_DIRECTORY, - overrides=overrides, - ) + overrides = ['+turn_timeout=300'] + + self._set_up_config(task_directory=TASK_DIRECTORY, overrides=overrides) # Set up the operator and server teacher = get_teacher(self.config) diff --git a/tests/crowdsourcing/tasks/test_chat_demo.py b/tests/crowdsourcing/tasks/test_chat_demo.py index c494233fb8d..880b683c535 100644 --- a/tests/crowdsourcing/tasks/test_chat_demo.py +++ b/tests/crowdsourcing/tasks/test_chat_demo.py @@ -381,9 +381,6 @@ def test_base_task(self): # Set up the config and database overrides = [ - '++mephisto.blueprint.world_file=${task_dir}/demo_worlds.py', - '++mephisto.blueprint.task_description_file=${task_dir}/task_description.html', - '++mephisto.blueprint.num_conversations=1', '++mephisto.task.allowed_concurrent=0', '++mephisto.task.assignment_duration_in_seconds=600', '++mephisto.task.max_num_concurrent_units=0', @@ -391,13 +388,7 @@ def test_base_task(self): '++num_turns=3', '++turn_timeout=300', ] - # TODO: remove all of these params once Hydra 1.1 is released with support - # for recursive defaults - self._set_up_config( - blueprint_type=BLUEPRINT_TYPE, - task_directory=TASK_DIRECTORY, - overrides=overrides, - ) + self._set_up_config(task_directory=TASK_DIRECTORY, overrides=overrides) # Set up the operator and server world_opt = { diff --git a/tests/crowdsourcing/tasks/turn_annotations_static/test_turn_annotations_static.py b/tests/crowdsourcing/tasks/turn_annotations_static/test_turn_annotations_static.py index 16481547dc3..90b6a907571 100644 --- a/tests/crowdsourcing/tasks/turn_annotations_static/test_turn_annotations_static.py +++ b/tests/crowdsourcing/tasks/turn_annotations_static/test_turn_annotations_static.py @@ -20,15 +20,12 @@ os.path.dirname(os.path.abspath(__file__)), 'task_config' ) TASK_DATA_FOLDER = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'task_data') +IN_FLIGHT_CONFIG_NAME = "example_in_flight_qa" try: from parlai.crowdsourcing.tasks.turn_annotations_static.run import TASK_DIRECTORY - from parlai.crowdsourcing.tasks.turn_annotations_static.turn_annotations_blueprint import ( - STATIC_BLUEPRINT_TYPE, - STATIC_IN_FLIGHT_QA_BLUEPRINT_TYPE, - ) from parlai.crowdsourcing.utils.frontend import build_task from parlai.crowdsourcing.utils.tests import AbstractOneTurnCrowdsourcingTest @@ -57,11 +54,9 @@ def test_no_in_flight_qa( self.operator = setup_teardown overrides = [ - '+mephisto.blueprint.annotation_indices_jsonl=null', - f'mephisto.blueprint.data_jsonl={TASK_CONFIG_FOLDER}/sample_conversations.jsonl', + f'mephisto.blueprint.data_jsonl={TASK_CONFIG_FOLDER}/sample_conversations.jsonl' ] self._test_turn_annotations_static_task( - blueprint_type=STATIC_BLUEPRINT_TYPE, task_data_path=os.path.join(TASK_DATA_FOLDER, 'no_in_flight_qa.json'), overrides=overrides, data_regression=data_regression, @@ -77,15 +72,13 @@ def test_in_flight_qa( self.operator = setup_teardown overrides = [ - '+mephisto.blueprint.annotation_indices_jsonl=null', - f'mephisto.blueprint.data_jsonl={TASK_CONFIG_FOLDER}/sample_conversations.jsonl', - f'+mephisto.blueprint.onboarding_in_flight_data={TASK_DIRECTORY}/task_config/onboarding_in_flight.jsonl', + f'mephisto.blueprint.data_jsonl={TASK_CONFIG_FOLDER}/sample_conversations.jsonl' ] self._test_turn_annotations_static_task( - blueprint_type=STATIC_IN_FLIGHT_QA_BLUEPRINT_TYPE, task_data_path=os.path.join(TASK_DATA_FOLDER, 'in_flight_qa.json'), overrides=overrides, data_regression=data_regression, + config_name=IN_FLIGHT_CONFIG_NAME, ) def test_in_flight_qa_annotation_file( @@ -103,24 +96,23 @@ def test_in_flight_qa_annotation_file( overrides = [ f'+mephisto.blueprint.annotation_indices_jsonl={TASK_DIRECTORY}/task_config/annotation_indices_example.jsonl', f'mephisto.blueprint.data_jsonl={TASK_CONFIG_FOLDER}/sample_conversations_annotation_file.jsonl', - f'+mephisto.blueprint.onboarding_in_flight_data={TASK_DIRECTORY}/task_config/onboarding_in_flight.jsonl', 'mephisto.blueprint.subtasks_per_unit=4', ] self._test_turn_annotations_static_task( - blueprint_type=STATIC_IN_FLIGHT_QA_BLUEPRINT_TYPE, task_data_path=os.path.join( TASK_DATA_FOLDER, 'in_flight_qa_annotation_file.json' ), overrides=overrides, data_regression=data_regression, + config_name=IN_FLIGHT_CONFIG_NAME, ) def _test_turn_annotations_static_task( self, - blueprint_type: str, task_data_path: str, overrides: List[str], data_regression: DataRegressionFixture, + config_name: str = 'example', ): """ Test the static turn annotations task under specific conditions. @@ -137,21 +129,10 @@ def _test_turn_annotations_static_task( build_task(task_directory=TASK_DIRECTORY) - # Set up the config and database - overrides += [ - '++mephisto.blueprint.annotation_last_only=False', - '++mephisto.blueprint.conversation_count=null', - '++mephisto.blueprint.onboarding_qualification=test_turn_annotations', - '++mephisto.blueprint.random_seed=42', - '++mephisto.task.assignment_duration_in_seconds=1800', - ] - # TODO: remove all of these params once Hydra 1.1 is released with support - # for recursive defaults - self._set_up_config( - blueprint_type=blueprint_type, task_directory=TASK_DIRECTORY, overrides=overrides, + config_name=config_name, ) # Set up the operator and server From 7db8f2483e5d24710a2cc5c6e7b2f428f17fa25c Mon Sep 17 00:00:00 2001 From: meganung Date: Wed, 8 Dec 2021 21:30:04 -0800 Subject: [PATCH 2/5] model chat remove overrides not needed --- .../tasks/model_chat/test_model_chat.py | 16 ---------------- .../tasks/model_chat/test_model_image_chat.py | 12 ------------ 2 files changed, 28 deletions(-) diff --git a/tests/crowdsourcing/tasks/model_chat/test_model_chat.py b/tests/crowdsourcing/tasks/model_chat/test_model_chat.py index 75f252de8d6..b77161f0ef5 100644 --- a/tests/crowdsourcing/tasks/model_chat/test_model_chat.py +++ b/tests/crowdsourcing/tasks/model_chat/test_model_chat.py @@ -110,27 +110,11 @@ def test_base_task(self): # Set up the config and database num_convos = 10 - args = ModelChatBlueprintArgs() overrides = [ - f'++mephisto.blueprint.{key}={val}' - for key, val in args.__dict__.items() - if key - in [ - 'max_onboard_time', - 'max_resp_time', - 'override_opt', - 'random_seed', - 'world_file', - ] - ] + [ - 'mephisto.blueprint.annotations_config_path=${task_dir}/task_config/annotations_config.json', f'mephisto.blueprint.conversations_needed_string=\"fixed_response:{num_convos:d}\"', f'mephisto.blueprint.chat_data_folder={chat_data_folder}', f'mephisto.blueprint.model_opt_path={model_opt_path}', - f'+mephisto.blueprint.num_conversations={num_convos:d}', ] - # TODO: remove all of these params once Hydra 1.1 is released with - # support for recursive defaults self._set_up_config(task_directory=TASK_DIRECTORY, overrides=overrides) # Set up the operator and server diff --git a/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py b/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py index a1d4e6570e7..1624780f92d 100644 --- a/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py +++ b/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py @@ -118,19 +118,7 @@ def test_base_task(self): # Set up the config and database num_convos = 1 - args = ModelImageChatBlueprintArgs() overrides = [ - f'++mephisto.blueprint.{key}={val}' - for key, val in args.__dict__.items() - if key - in [ - 'evals_per_image_model_combo', - 'max_resp_time', - 'override_opt', - 'random_seed', - 'world_file', - ] - ] + [ f'mephisto.blueprint.chat_data_folder={chat_data_folder}', f'mephisto.blueprint.image_context_path={image_context_path}', f'mephisto.blueprint.num_conversations={num_convos:d}', From ac35ed55cd326865dd3da378373258c554f2f16e Mon Sep 17 00:00:00 2001 From: meganung Date: Thu, 9 Dec 2021 09:45:37 -0800 Subject: [PATCH 3/5] remove unneeded override --- tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py b/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py index 1624780f92d..a6cd1c4530a 100644 --- a/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py +++ b/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py @@ -123,7 +123,6 @@ def test_base_task(self): f'mephisto.blueprint.image_context_path={image_context_path}', f'mephisto.blueprint.num_conversations={num_convos:d}', f'mephisto.blueprint.stack_folder={stack_folder}', - 'mephisto.blueprint.task_model_parallel=False', ] self._set_up_config( From 0da66bbac21ea030c6fe916157009c822204fe7e Mon Sep 17 00:00:00 2001 From: meganung Date: Thu, 9 Dec 2021 10:55:29 -0800 Subject: [PATCH 4/5] remove todo that has been addressed --- parlai/crowdsourcing/utils/tests.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/parlai/crowdsourcing/utils/tests.py b/parlai/crowdsourcing/utils/tests.py index 2836d0597aa..b769fc9133e 100644 --- a/parlai/crowdsourcing/utils/tests.py +++ b/parlai/crowdsourcing/utils/tests.py @@ -96,13 +96,6 @@ def _set_up_config( ] + overrides, ) - # TODO: when Hydra 1.1 is released with support for recursive defaults, - # don't manually specify all missing blueprint args anymore, but - # instead define the blueprint in the defaults list directly. - # Currently, the blueprint can't be set in the defaults list without - # overriding params in the YAML file, as documented at - # https://github.com/facebookresearch/hydra/issues/326 and as fixed in - # https://github.com/facebookresearch/hydra/pull/1044. self.data_dir = tempfile.mkdtemp() self.database_path = os.path.join(self.data_dir, "mephisto.db") From 768e5608503a7108c35bba781a6baab63f4166b0 Mon Sep 17 00:00:00 2001 From: meganung Date: Thu, 9 Dec 2021 11:45:09 -0800 Subject: [PATCH 5/5] remove blueprint imports --- tests/crowdsourcing/tasks/model_chat/test_model_chat.py | 2 -- tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py | 2 -- tests/crowdsourcing/tasks/test_chat_demo.py | 1 - 3 files changed, 5 deletions(-) diff --git a/tests/crowdsourcing/tasks/model_chat/test_model_chat.py b/tests/crowdsourcing/tasks/model_chat/test_model_chat.py index b77161f0ef5..8cac9644f24 100644 --- a/tests/crowdsourcing/tasks/model_chat/test_model_chat.py +++ b/tests/crowdsourcing/tasks/model_chat/test_model_chat.py @@ -63,8 +63,6 @@ from parlai.crowdsourcing.tasks.model_chat.run import TASK_DIRECTORY from parlai.crowdsourcing.tasks.model_chat.model_chat_blueprint import ( SharedModelChatTaskState, - ModelChatBlueprintArgs, - BLUEPRINT_TYPE, ) from parlai.crowdsourcing.tasks.model_chat.utils import AbstractModelChatTest diff --git a/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py b/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py index a6cd1c4530a..076d0f1de32 100644 --- a/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py +++ b/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py @@ -44,8 +44,6 @@ from parlai.crowdsourcing.tasks.model_chat.run_image_chat import TASK_DIRECTORY from parlai.crowdsourcing.tasks.model_chat.model_chat_blueprint import ( SharedModelImageChatTaskState, - ModelImageChatBlueprintArgs, - IMAGE_CHAT_BLUEPRINT_TYPE, ) from parlai.crowdsourcing.tasks.model_chat.utils import AbstractModelChatTest diff --git a/tests/crowdsourcing/tasks/test_chat_demo.py b/tests/crowdsourcing/tasks/test_chat_demo.py index 880b683c535..4f60891e843 100644 --- a/tests/crowdsourcing/tasks/test_chat_demo.py +++ b/tests/crowdsourcing/tasks/test_chat_demo.py @@ -347,7 +347,6 @@ import mephisto from mephisto.abstractions.blueprints.parlai_chat.parlai_chat_blueprint import ( SharedParlAITaskState, - BLUEPRINT_TYPE, ) from parlai.crowdsourcing.utils.tests import AbstractParlAIChatTest