Tests for the static turn annotations tasks

parlai/crowdsourcing/tasks/turn_annotations_static/util.py

@@ -21,9 +21,6 @@ def run_static_task(cfg: DictConfig, task_directory: str):
     Run static task, given configuration.
     """
 
-    frontend_source_dir = os.path.join(task_directory, "webapp")
-    frontend_build_dir = os.path.join(frontend_source_dir, "build")
-
     db, cfg = load_db_and_process_config(cfg)
     print(f'\nHydra config:\n{OmegaConf.to_yaml(cfg)}')
 
@@ -33,8 +30,26 @@ def run_static_task(cfg: DictConfig, task_directory: str):
     # Default to a task-specific name to avoid soft-block collisions
     soft_block_mturk_workers(cfg=cfg, db=db, soft_block_qual_name=soft_block_qual_name)
 
-    # Build the task
+    build_task(task_directory)
+
+    operator = Operator(db)
+    operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=None)
+    operator.wait_for_runs_then_shutdown(
+        skip_input=True, log_rate=cfg.monitoring_log_rate
+    )
+
+
+def build_task(task_directory: str):
+    """
+    Build the task with npm.
+    """
+
+    # Paths
+    frontend_source_dir = os.path.join(task_directory, "webapp")
+    frontend_build_dir = os.path.join(frontend_source_dir, "build")
     return_dir = os.getcwd()
+
+    # Build the task
     os.chdir(frontend_source_dir)
     if os.path.exists(frontend_build_dir):
         shutil.rmtree(frontend_build_dir)
@@ -51,9 +66,3 @@ def run_static_task(cfg: DictConfig, task_directory: str):
             "frontend. See the above error for more information."
         )
     os.chdir(return_dir)
-
-    operator = Operator(db)
-    operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=None)
-    operator.wait_for_runs_then_shutdown(
-        skip_input=True, log_rate=cfg.monitoring_log_rate
-    )

parlai/crowdsourcing/utils/tests.py

@@ -10,7 +10,8 @@
 import os
 import tempfile
 import time
-from typing import List, Optional
+import unittest
+from typing import Any, Dict, List, Optional
 
 from hydra.experimental import compose, initialize
 from mephisto.core.local_database import LocalMephistoDB
@@ -19,9 +20,9 @@
 from mephisto.utils.scripts import augment_config_from_db
 
 
-class CrowdsourcingTestMixin:
+class AbstractCrowdsourcingTest(unittest.TestCase):
     """
-    Mixin for end-to-end tests of Mephisto-based crowdsourcing tasks.
+    Abstract class for end-to-end tests of Mephisto-based crowdsourcing tasks.
 
     Allows for setup and teardown of the operator, as well as for config specification
     and agent registration.
@@ -119,3 +120,39 @@ def _register_mock_agents(self, num_agents: int = 1) -> List[str]:
         agent_ids = [agent.db_id for agent in agents]
 
         return agent_ids
+
+
+class AbstractOneTurnCrowdsourcingTest(AbstractCrowdsourcingTest):
+    """
+    Abstract class for end-to-end tests of one-turn crowdsourcing tasks.
+
+    Useful for Blueprints such as AcuteEvalBlueprint and StaticReactBlueprint for which
+    all of the worker's responses are sent to the backend code at once.
+    """
+
+    def _test_agent_state(self, expected_state: Dict[str, Any]):
+        """
+        Test that the actual agent state matches the expected state.
+
+        Register a mock human agent, request initial data to define the 'inputs' field
+        of the agent state, make the agent act to define the 'outputs' field of the
+        agent state, and then check that the agent state matches the desired agent
+        state.
+        """
+
+        # Set up the mock human agent
+        agent_id = self._register_mock_agents(num_agents=1)[0]
+
+        # Set initial data
+        self.server.request_init_data(agent_id)
+
+        # Make agent act
+        self.server.send_agent_act(
+            agent_id,
+            {"MEPHISTO_is_submit": True, "task_data": expected_state['outputs']},
+        )
+
+        # Check that the inputs and outputs are as expected
+        state = self.db.find_agents()[0].state.get_data()
+        self.assertEqual(expected_state['inputs'], state['inputs'])
+        self.assertEqual(expected_state['outputs'], state['outputs'])

tests/crowdsourcing/tasks/test_acute_eval.py

@@ -256,9 +256,9 @@
         BLUEPRINT_TYPE,
     )
     from parlai.crowdsourcing.tasks.acute_eval.run import TASK_DIRECTORY
-    from parlai.crowdsourcing.utils.tests import CrowdsourcingTestMixin
+    from parlai.crowdsourcing.utils.tests import AbstractOneTurnCrowdsourcingTest
 
-    class TestAcuteEval(CrowdsourcingTestMixin, unittest.TestCase):
+    class TestAcuteEval(AbstractOneTurnCrowdsourcingTest):
         """
         Test the ACUTE-Eval crowdsourcing task.
         """
@@ -274,21 +274,9 @@ def test_base_task(self):
             )
             self._set_up_server()
 
-            # Set up the mock human agent
-            agent_id = self._register_mock_agents(num_agents=1)[0]
-
-            # Set initial data
-            self.server.request_init_data(agent_id)
-
-            # Make agent act
-            self.server.send_agent_act(
-                agent_id, {"MEPHISTO_is_submit": True, "task_data": DESIRED_OUTPUTS}
-            )
-
-            # Check that the inputs and outputs are as expected
-            state = self.db.find_agents()[0].state.get_data()
-            self.assertEqual(DESIRED_INPUTS, state['inputs'])
-            self.assertEqual(DESIRED_OUTPUTS, state['outputs'])
+            # Check that the agent state is as it should be
+            expected_state = {'inputs': DESIRED_INPUTS, 'outputs': DESIRED_OUTPUTS}
+            self._test_agent_state(expected_state=expected_state)
 
 
 except ImportError:

tests/crowdsourcing/tasks/test_chat_demo.py

@@ -7,6 +7,8 @@
 End-to-end testing for the chat demo crowdsourcing task.
 """
 
+import os
+import sys
 import unittest
 
 # Desired inputs/outputs
@@ -339,16 +341,22 @@
 
 try:
 
-    # From the Mephisto repo
-    from examples.parlai_chat_task_demo.parlai_test_script import TASK_DIRECTORY
+    import mephisto
     from mephisto.server.blueprints.parlai_chat.parlai_chat_blueprint import (
         SharedParlAITaskState,
         BLUEPRINT_TYPE,
     )
 
-    from parlai.crowdsourcing.utils.tests import CrowdsourcingTestMixin
+    # From the Mephisto repo
+    mephisto_repo_folder = os.path.dirname(
+        os.path.dirname(os.path.abspath(mephisto.__file__))
+    )
+    sys.path.insert(1, mephisto_repo_folder)
+    from examples.parlai_chat_task_demo.parlai_test_script import TASK_DIRECTORY
+
+    from parlai.crowdsourcing.utils.tests import AbstractCrowdsourcingTest
 
-    class TestChatDemo(CrowdsourcingTestMixin, unittest.TestCase):
+    class TestChatDemo(AbstractCrowdsourcingTest):
         """
         Test the chat demo crowdsourcing task.
         """