Warning on unrecommended ACUTE-Eval value (#3536)

* Check max units per worker

* Change warning

* Reword

parlai/crowdsourcing/tasks/acute_eval/acute_eval_blueprint.py

@@ -9,16 +9,17 @@
 from dataclasses import dataclass, field
 from typing import ClassVar, List, Type, Any, Dict, Iterable, TYPE_CHECKING
 
-from mephisto.operations.registry import register_mephisto_abstraction
 from mephisto.abstractions.blueprint import Blueprint, BlueprintArgs, SharedTaskState
 from mephisto.data_model.assignment import InitializationData
+from mephisto.operations.registry import register_mephisto_abstraction
 from omegaconf import MISSING, DictConfig
 
 from parlai.crowdsourcing.tasks.acute_eval.acute_eval_agent_state import (
     AcuteEvalAgentState,
 )
 from parlai.crowdsourcing.tasks.acute_eval.acute_eval_builder import AcuteEvalBuilder
 from parlai.crowdsourcing.tasks.acute_eval.acute_eval_runner import AcuteEvalRunner
+from parlai.utils.misc import warn_once
 
 if TYPE_CHECKING:
     from mephisto.abstractions.blueprint import AgentState, TaskRunner, TaskBuilder
@@ -96,7 +97,10 @@ class AcuteEvalBlueprint(Blueprint):
     @classmethod
     def assert_task_args(cls, args: DictConfig, shared_state: SharedTaskState) -> None:
         """
-        Ensure that the data can be properly loaded.
+        ACUTE-Eval-specific argument checking.
+
+        Ensure that the data can be properly loaded and that recommended worker settings
+        are used.
         """
         if args.blueprint.get("pairings_filepath", None) is not None:
             pairings_filepath = os.path.expanduser(args.blueprint.pairings_filepath)
@@ -114,6 +118,15 @@ def assert_task_args(cls, args: DictConfig, shared_state: SharedTaskState) -> No
                     "Must provide `block_qualification` to use `block_on_onboarding_fail`"
                 )
 
+        if args.task.get('maximum_units_per_worker', None) != 1:
+            warn_once(
+                'It is *strongly* recommended to use a '
+                'args.task.maximum_units_per_worker value of 1, as was done in the '
+                'ACUTE-Eval paper! Anything else will not provide consistent results '
+                'as quoted there. Go to ParlAI\'s GitHub Discussion page if you\'d '
+                'like to discuss further.'
+            )
+
     def get_frontend_args(self) -> Dict[str, Any]:
         """
         Specifies what options within a task_config should be forwarded to the client