From 26ea1b857b2ca261b81637214fc0b19344873b55 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Thu, 23 Dec 2021 11:52:34 -0500 Subject: [PATCH 1/5] Layer changes --- .../tasks/model_chat/frontend/main.js | 17 +++++++++----- .../crowdsourcing/tasks/model_chat/worlds.py | 23 ++++++++++++++----- .../tasks/model_chat/worlds_image_chat.py | 2 -- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/parlai/crowdsourcing/tasks/model_chat/frontend/main.js b/parlai/crowdsourcing/tasks/model_chat/frontend/main.js index 0c66c2861b2..af829468664 100644 --- a/parlai/crowdsourcing/tasks/model_chat/frontend/main.js +++ b/parlai/crowdsourcing/tasks/model_chat/frontend/main.js @@ -31,7 +31,13 @@ function MainApp() { renderSidePane={({ mephistoContext: { taskConfig }, appContext: { taskContext } }) => ( {(taskContext.hasOwnProperty('image_src') && taskContext['image_src']) ? (
@@ -45,24 +51,23 @@ function MainApp() { )} renderTextResponse={ - ({ - mephistoContext: { taskConfig }, + ({ + mephistoContext: { taskConfig }, appContext: { appSettings }, onMessageSend, active, }) => ( - - ) + ) } /> ); } ReactDOM.render(, document.getElementById("app")); - diff --git a/parlai/crowdsourcing/tasks/model_chat/worlds.py b/parlai/crowdsourcing/tasks/model_chat/worlds.py index 2f819d9409e..3585e201258 100644 --- a/parlai/crowdsourcing/tasks/model_chat/worlds.py +++ b/parlai/crowdsourcing/tasks/model_chat/worlds.py @@ -180,6 +180,9 @@ def __init__(self, opt, agent, bot): self.task_turn_idx = 0 self.num_turns = num_turns + self.agent.agent_id = 'Speaker 1' + self.bot.agent_id = 'Speaker 2' + self.dialog = [] self.tag = f'conversation_id {agent.mephisto_agent.db_id}' self.task_type = 'sandbox' if opt['is_sandbox'] else 'live' @@ -488,7 +491,7 @@ def _run_initial_turn(self) -> None: # Display the previous two utterances human_first_msg = { 'episode_done': False, - 'id': self.agent.id, + 'id': self.agent.agent_id, 'text': self.context_info['person1_seed_utterance'], 'fake_start': True, 'agent_idx': 0, @@ -497,7 +500,7 @@ def _run_initial_turn(self) -> None: human_first_msg[k] = v bot_first_msg = { 'episode_done': False, - 'id': self.bot.id, + 'id': self.bot.agent_id, 'text': self.context_info['person2_seed_utterance'], 'fake_start': True, 'agent_idx': 1, @@ -513,12 +516,20 @@ def _run_initial_turn(self) -> None: elif self.opt['conversation_start_mode'] == 'hi': print('[Displaying "Hi!" only as per Meena task.]') + if self.personas is not None: + human_persona_strings = [s.strip() for s in self.personas[0]] + else: + human_persona_strings = ['', ''] human_first_msg = { 'episode_done': False, - 'id': self.agent.id, + 'id': self.agent.agent_id, 'text': 'Hi!', 'fake_start': True, 'agent_idx': 0, + 'task_data': { + 'human_persona_string_1': human_persona_strings[0], + 'human_persona_string_2': human_persona_strings[1], + }, } for k, v in control_msg.items(): human_first_msg[k] = v @@ -528,7 +539,9 @@ def _run_initial_turn(self) -> None: self.bot.observe(validate(human_first_msg)) first_bot_act = self.bot.act() - first_bot_act = Compatibility.maybe_fix_act(first_bot_act) + first_bot_act = Compatibility.backward_compatible_force_set( + first_bot_act, 'id', self.bot.agent_id + ) self.agent.observe(validate(first_bot_act)) @@ -671,8 +684,6 @@ def make_world(opt, agents): statistics_condition = opt['statistics_condition'] context_generator = opt['context_generator'] - agents[0].agent_id = "Worker" - # Get context: personas, previous utterances, etc. if context_generator is not None: context_info = context_generator.get_context() diff --git a/parlai/crowdsourcing/tasks/model_chat/worlds_image_chat.py b/parlai/crowdsourcing/tasks/model_chat/worlds_image_chat.py index b96931d81bb..138d2b244de 100644 --- a/parlai/crowdsourcing/tasks/model_chat/worlds_image_chat.py +++ b/parlai/crowdsourcing/tasks/model_chat/worlds_image_chat.py @@ -134,8 +134,6 @@ def shutdown(self): def make_world(opt, agents): - agents[0].agent_id = "Worker" - # We are showing an image to the worker and bot, so grab the image path and other # context info image_idx, model_name, no_more_work = opt['image_stack'].get_next_image( From d625a2634157dfe04d8f29821b617424b4599aa2 Mon Sep 17 00:00:00 2001 From: EricMichaelSmith Date: Thu, 23 Dec 2021 14:02:46 -0500 Subject: [PATCH 2/5] Fixes --- parlai/crowdsourcing/tasks/model_chat/run.py | 3 +-- parlai/crowdsourcing/tasks/model_chat/worlds_image_chat.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parlai/crowdsourcing/tasks/model_chat/run.py b/parlai/crowdsourcing/tasks/model_chat/run.py index 70f17f9a2f6..8e0a3c04526 100644 --- a/parlai/crowdsourcing/tasks/model_chat/run.py +++ b/parlai/crowdsourcing/tasks/model_chat/run.py @@ -14,7 +14,6 @@ from parlai.crowdsourcing.tasks.model_chat.impl import run_task from parlai.crowdsourcing.utils.mturk import MTurkRunScriptConfig -import parlai.crowdsourcing.tasks.model_chat.worlds as world_module TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) @@ -40,7 +39,7 @@ class ScriptConfig(MTurkRunScriptConfig): @hydra.main(config_path="hydra_configs", config_name="scriptconfig") def main(cfg: DictConfig) -> None: - run_task(cfg=cfg, task_directory=TASK_DIRECTORY, world_module=world_module) + run_task(cfg=cfg, task_directory=TASK_DIRECTORY) if __name__ == "__main__": diff --git a/parlai/crowdsourcing/tasks/model_chat/worlds_image_chat.py b/parlai/crowdsourcing/tasks/model_chat/worlds_image_chat.py index 138d2b244de..65a45827c8a 100644 --- a/parlai/crowdsourcing/tasks/model_chat/worlds_image_chat.py +++ b/parlai/crowdsourcing/tasks/model_chat/worlds_image_chat.py @@ -78,6 +78,7 @@ def _run_initial_turn(self) -> None: bot_first_act_raw = Message( Compatibility.maybe_fix_act(bot_first_act_raw) ).json_safe_payload() + bot_first_act_raw['id'] = self.bot.agent_id self.agent.observe(validate(bot_first_act_raw)) bot_first_act = { 'episode_done': False, From 23871df9d23f57a78a109543af03c6f3d4fcf9ef Mon Sep 17 00:00:00 2001 From: EricMichaelSmith Date: Thu, 23 Dec 2021 14:22:50 -0500 Subject: [PATCH 3/5] Work on CI checks --- .../expected_states/final_chat_data.json | 24 +++--- .../final_chat_data__image_chat.json | 26 +++--- .../model_chat/expected_states/state.json | 76 ++++++++--------- .../expected_states/state__image_chat.json | 82 +++++++++---------- .../tasks/model_chat/test_model_chat.py | 2 +- .../tasks/model_chat/test_model_image_chat.py | 2 +- 6 files changed, 106 insertions(+), 106 deletions(-) diff --git a/tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data.json b/tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data.json index a8d43fb8bbd..0fb70e37045 100644 --- a/tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data.json +++ b/tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data.json @@ -7,7 +7,7 @@ "dialog": [ { "episode_done": false, - "id": "Worker", + "id": "Speaker 1", "text": "Hi!", "fake_start": true, "agent_idx": 0, @@ -16,7 +16,7 @@ { "agent_idx": 1, "text": "I don't know.", - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, @@ -29,12 +29,12 @@ { "agent_idx": 0, "text": "What are you nervous about?", - "id": "Worker" + "id": "Speaker 1" }, { "agent_idx": 1, "text": "I don't know.", - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, @@ -47,12 +47,12 @@ { "agent_idx": 0, "text": "Do you have any plans for the weekend?", - "id": "Worker" + "id": "Speaker 1" }, { "agent_idx": 1, "text": "I don't know.", - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, @@ -65,12 +65,12 @@ { "agent_idx": 0, "text": "Yeah that sounds great! I like to bike and try new restaurants.", - "id": "Worker" + "id": "Speaker 1" }, { "agent_idx": 1, "text": "I don't know.", - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, @@ -83,12 +83,12 @@ { "agent_idx": 0, "text": "Oh, Italian food is great. I also love Thai and Indian.", - "id": "Worker" + "id": "Speaker 1" }, { "agent_idx": 1, "text": "I don't know.", - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, @@ -101,12 +101,12 @@ { "agent_idx": 0, "text": "Hmmm - anything with peanuts? Or I like when they have spicy licorice-like herbs.", - "id": "Worker" + "id": "Speaker 1" }, { "agent_idx": 1, "text": "I don't know.", - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, diff --git a/tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data__image_chat.json b/tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data__image_chat.json index f64c2636e7f..153ada54afd 100644 --- a/tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data__image_chat.json +++ b/tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data__image_chat.json @@ -16,69 +16,69 @@ }, { "episode_done": false, - "id": "TransresnetAgent", + "id": "Speaker 2", "text": "I must learn that bird's name!", "agent_idx": 1 }, { "agent_idx": 0, "text": "Response 1", - "id": "Worker" + "id": "Speaker 1" }, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent" + "id": "Speaker 2" }, { "agent_idx": 0, "text": "Response 2", - "id": "Worker" + "id": "Speaker 1" }, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent" + "id": "Speaker 2" }, { "agent_idx": 0, "text": "Response 3", - "id": "Worker" + "id": "Speaker 1" }, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent" + "id": "Speaker 2" }, { "agent_idx": 0, "text": "Response 4", - "id": "Worker" + "id": "Speaker 1" }, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent" + "id": "Speaker 2" }, { "agent_idx": 0, "text": "Response 5", - "id": "Worker" + "id": "Speaker 1" }, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent" + "id": "Speaker 2" }, { "agent_idx": 0, "text": "Response 6", - "id": "Worker" + "id": "Speaker 1" }, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent", + "id": "Speaker 2", "final_rating": 0 } ], diff --git a/tests/crowdsourcing/tasks/model_chat/expected_states/state.json b/tests/crowdsourcing/tasks/model_chat/expected_states/state.json index 8f4d83c082c..87f7885c653 100644 --- a/tests/crowdsourcing/tasks/model_chat/expected_states/state.json +++ b/tests/crowdsourcing/tasks/model_chat/expected_states/state.json @@ -5,7 +5,7 @@ { "data": { "message_id": "3a289317-8519-4da8-be8f-9a515a6ec3da", - "state": {"agent_display_name": "Worker"} + "state": {"agent_display_name": "Speaker 1"} }, "packet_type": "update_status", "receiver_id": "1", @@ -17,7 +17,7 @@ "agent_idx": 0, "episode_done": false, "fake_start": true, - "id": "Worker", + "id": "Speaker 1", "message_id": "4f72800c-e1ff-4411-8ee5-286336c27859", "text": "Hi!" }, @@ -29,7 +29,7 @@ { "data": { "episode_done": false, - "id": "FixedResponseAgent", + "id": "Speaker 2", "message_id": "43ef23af-a9fc-4f67-bd06-6e083ba45678", "text": "I don't know." }, @@ -41,7 +41,7 @@ { "data": { "episode_done": false, - "id": "Worker", + "id": "Speaker 1", "message_id": "fb168956-cfe1-40e2-b28b-1ca66207a39e", "task_data": { "problem_data_for_prior_message": { @@ -63,7 +63,7 @@ { "data": { "episode_done": false, - "id": "FixedResponseAgent", + "id": "Speaker 2", "message_id": "05fefda5-dfa0-44e2-81ad-ba9ca203ed1d", "text": "I don't know." }, @@ -75,7 +75,7 @@ { "data": { "episode_done": false, - "id": "Worker", + "id": "Speaker 1", "message_id": "49740817-1443-47b4-b91d-3260fd12bf7d", "task_data": { "problem_data_for_prior_message": { @@ -97,7 +97,7 @@ { "data": { "episode_done": false, - "id": "FixedResponseAgent", + "id": "Speaker 2", "message_id": "6a52d8ee-1955-4d08-a7f8-007feea3ecfd", "text": "I don't know." }, @@ -109,7 +109,7 @@ { "data": { "episode_done": false, - "id": "Worker", + "id": "Speaker 1", "message_id": "db9a475a-a47f-4328-92f4-130f2227dfbb", "task_data": { "problem_data_for_prior_message": { @@ -131,7 +131,7 @@ { "data": { "episode_done": false, - "id": "FixedResponseAgent", + "id": "Speaker 2", "message_id": "550f71a4-f441-4546-bd73-45313932ec3c", "text": "I don't know." }, @@ -143,7 +143,7 @@ { "data": { "episode_done": false, - "id": "Worker", + "id": "Speaker 1", "message_id": "9dc9241c-73f6-4724-bad6-50038ab64522", "task_data": { "problem_data_for_prior_message": { @@ -165,7 +165,7 @@ { "data": { "episode_done": false, - "id": "FixedResponseAgent", + "id": "Speaker 2", "message_id": "3a046372-6204-452d-8877-88a1d1a9583a", "text": "I don't know." }, @@ -177,7 +177,7 @@ { "data": { "episode_done": false, - "id": "Worker", + "id": "Speaker 1", "message_id": "64816a92-5e8a-4ab9-b194-2a93499f47dc", "task_data": { "problem_data_for_prior_message": { @@ -199,7 +199,7 @@ { "data": { "episode_done": false, - "id": "FixedResponseAgent", + "id": "Speaker 2", "message_id": "f68faec0-b2c2-434d-98a6-b4cc9f1f42b3", "text": "I don't know." }, @@ -211,7 +211,7 @@ { "data": { "episode_done": false, - "id": "Worker", + "id": "Speaker 1", "message_id": "524d5a56-71a5-4b00-8595-0df720004eb6", "task_data": { "final_rating": 4, @@ -243,13 +243,13 @@ "agent_idx": 0, "episode_done": false, "fake_start": true, - "id": "Worker", + "id": "Speaker 1", "message_id": "4f72800c-e1ff-4411-8ee5-286336c27859", "text": "Hi!" }, { "agent_idx": 1, - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, @@ -262,12 +262,12 @@ }, { "agent_idx": 0, - "id": "Worker", + "id": "Speaker 1", "text": "What are you nervous about?" }, { "agent_idx": 1, - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, @@ -280,12 +280,12 @@ }, { "agent_idx": 0, - "id": "Worker", + "id": "Speaker 1", "text": "Do you have any plans for the weekend?" }, { "agent_idx": 1, - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, @@ -298,12 +298,12 @@ }, { "agent_idx": 0, - "id": "Worker", + "id": "Speaker 1", "text": "Yeah that sounds great! I like to bike and try new restaurants." }, { "agent_idx": 1, - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, @@ -316,12 +316,12 @@ }, { "agent_idx": 0, - "id": "Worker", + "id": "Speaker 1", "text": "Oh, Italian food is great. I also love Thai and Indian." }, { "agent_idx": 1, - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, @@ -334,12 +334,12 @@ }, { "agent_idx": 0, - "id": "Worker", + "id": "Speaker 1", "text": "Hmmm - anything with peanuts? Or I like when they have spicy licorice-like herbs." }, { "agent_idx": 1, - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, @@ -399,7 +399,7 @@ "dialog": [ { "episode_done": false, - "id": "Worker", + "id": "Speaker 1", "text": "Hi!", "fake_start": true, "agent_idx": 0 @@ -407,7 +407,7 @@ { "agent_idx": 1, "text": "I don't know.", - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, @@ -420,12 +420,12 @@ { "agent_idx": 0, "text": "What are you nervous about?", - "id": "Worker" + "id": "Speaker 1" }, { "agent_idx": 1, "text": "I don't know.", - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, @@ -438,12 +438,12 @@ { "agent_idx": 0, "text": "Do you have any plans for the weekend?", - "id": "Worker" + "id": "Speaker 1" }, { "agent_idx": 1, "text": "I don't know.", - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, @@ -456,12 +456,12 @@ { "agent_idx": 0, "text": "Yeah that sounds great! I like to bike and try new restaurants.", - "id": "Worker" + "id": "Speaker 1" }, { "agent_idx": 1, "text": "I don't know.", - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, @@ -474,12 +474,12 @@ { "agent_idx": 0, "text": "Oh, Italian food is great. I also love Thai and Indian.", - "id": "Worker" + "id": "Speaker 1" }, { "agent_idx": 1, "text": "I don't know.", - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, @@ -492,12 +492,12 @@ { "agent_idx": 0, "text": "Hmmm - anything with peanuts? Or I like when they have spicy licorice-like herbs.", - "id": "Worker" + "id": "Speaker 1" }, { "agent_idx": 1, "text": "I don't know.", - "id": "FixedResponseAgent", + "id": "Speaker 2", "problem_data": { "bucket_0": false, "bucket_1": false, diff --git a/tests/crowdsourcing/tasks/model_chat/expected_states/state__image_chat.json b/tests/crowdsourcing/tasks/model_chat/expected_states/state__image_chat.json index f87a0bb081a..7b2dddec4b1 100644 --- a/tests/crowdsourcing/tasks/model_chat/expected_states/state__image_chat.json +++ b/tests/crowdsourcing/tasks/model_chat/expected_states/state__image_chat.json @@ -6,7 +6,7 @@ "sender_id": "mephisto", "receiver_id": "1", "data": { - "state": {"agent_display_name": "Worker"}, + "state": {"agent_display_name": "Speaker 1"}, "message_id": "79ee17f5-0edd-45ea-9e12-0f73782de2c4" }, "timestamp": 1609255262.574956 @@ -33,7 +33,7 @@ "receiver_id": "1", "data": { "text": "I must learn that bird's name!", - "id": "TransresnetAgent", + "id": "Speaker 2", "text_candidates": [ "I must learn that bird's name!", "My, aren't you a pretty bird?" @@ -50,7 +50,7 @@ "data": { "text": "Response 1", "task_data": {}, - "id": "Worker", + "id": "Speaker 1", "episode_done": false }, "timestamp": 1609255264.653284 @@ -61,7 +61,7 @@ "receiver_id": "1", "data": { "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent", + "id": "Speaker 2", "text_candidates": [ "My, aren't you a pretty bird?", "I must learn that bird's name!" @@ -78,7 +78,7 @@ "data": { "text": "Response 2", "task_data": {}, - "id": "Worker", + "id": "Speaker 1", "episode_done": false }, "timestamp": 1609255264.6686988 @@ -89,7 +89,7 @@ "receiver_id": "1", "data": { "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent", + "id": "Speaker 2", "text_candidates": [ "My, aren't you a pretty bird?", "I must learn that bird's name!" @@ -106,7 +106,7 @@ "data": { "text": "Response 3", "task_data": {}, - "id": "Worker", + "id": "Speaker 1", "episode_done": false }, "timestamp": 1609255264.6849954 @@ -117,7 +117,7 @@ "receiver_id": "1", "data": { "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent", + "id": "Speaker 2", "text_candidates": [ "My, aren't you a pretty bird?", "I must learn that bird's name!" @@ -134,7 +134,7 @@ "data": { "text": "Response 4", "task_data": {}, - "id": "Worker", + "id": "Speaker 1", "episode_done": false }, "timestamp": 1609255264.701345 @@ -145,7 +145,7 @@ "receiver_id": "1", "data": { "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent", + "id": "Speaker 2", "text_candidates": [ "My, aren't you a pretty bird?", "I must learn that bird's name!" @@ -162,7 +162,7 @@ "data": { "text": "Response 5", "task_data": {}, - "id": "Worker", + "id": "Speaker 1", "episode_done": false }, "timestamp": 1609255264.7193751 @@ -173,7 +173,7 @@ "receiver_id": "1", "data": { "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent", + "id": "Speaker 2", "text_candidates": [ "My, aren't you a pretty bird?", "I must learn that bird's name!" @@ -190,7 +190,7 @@ "data": { "text": "Response 6", "task_data": {}, - "id": "Worker", + "id": "Speaker 1", "episode_done": false }, "timestamp": 1609255264.7359936 @@ -201,7 +201,7 @@ "receiver_id": "1", "data": { "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent", + "id": "Speaker 2", "text_candidates": [ "My, aren't you a pretty bird?", "I must learn that bird's name!" @@ -218,7 +218,7 @@ "data": { "text": "", "task_data": {"final_rating": 0}, - "id": "Worker", + "id": "Speaker 1", "episode_done": false }, "timestamp": 1609255264.7529757 @@ -240,45 +240,45 @@ }, { "episode_done": false, - "id": "TransresnetAgent", + "id": "Speaker 2", "text": "I must learn that bird's name!", "agent_idx": 1 }, - {"agent_idx": 0, "text": "Response 1", "id": "Worker"}, + {"agent_idx": 0, "text": "Response 1", "id": "Speaker 1"}, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent" + "id": "Speaker 2" }, - {"agent_idx": 0, "text": "Response 2", "id": "Worker"}, + {"agent_idx": 0, "text": "Response 2", "id": "Speaker 1"}, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent" + "id": "Speaker 2" }, - {"agent_idx": 0, "text": "Response 3", "id": "Worker"}, + {"agent_idx": 0, "text": "Response 3", "id": "Speaker 1"}, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent" + "id": "Speaker 2" }, - {"agent_idx": 0, "text": "Response 4", "id": "Worker"}, + {"agent_idx": 0, "text": "Response 4", "id": "Speaker 1"}, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent" + "id": "Speaker 2" }, - {"agent_idx": 0, "text": "Response 5", "id": "Worker"}, + {"agent_idx": 0, "text": "Response 5", "id": "Speaker 1"}, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent" + "id": "Speaker 2" }, - {"agent_idx": 0, "text": "Response 6", "id": "Worker"}, + {"agent_idx": 0, "text": "Response 6", "id": "Speaker 1"}, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent", + "id": "Speaker 2", "final_rating": 0 } ], @@ -463,45 +463,45 @@ }, { "episode_done": false, - "id": "TransresnetAgent", + "id": "Speaker 2", "text": "I must learn that bird's name!", "agent_idx": 1 }, - {"agent_idx": 0, "text": "Response 1", "id": "Worker"}, + {"agent_idx": 0, "text": "Response 1", "id": "Speaker 1"}, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent" + "id": "Speaker 2" }, - {"agent_idx": 0, "text": "Response 2", "id": "Worker"}, + {"agent_idx": 0, "text": "Response 2", "id": "Speaker 1"}, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent" + "id": "Speaker 2" }, - {"agent_idx": 0, "text": "Response 3", "id": "Worker"}, + {"agent_idx": 0, "text": "Response 3", "id": "Speaker 1"}, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent" + "id": "Speaker 2" }, - {"agent_idx": 0, "text": "Response 4", "id": "Worker"}, + {"agent_idx": 0, "text": "Response 4", "id": "Speaker 1"}, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent" + "id": "Speaker 2" }, - {"agent_idx": 0, "text": "Response 5", "id": "Worker"}, + {"agent_idx": 0, "text": "Response 5", "id": "Speaker 1"}, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent" + "id": "Speaker 2" }, - {"agent_idx": 0, "text": "Response 6", "id": "Worker"}, + {"agent_idx": 0, "text": "Response 6", "id": "Speaker 1"}, { "agent_idx": 1, "text": "My, aren't you a pretty bird?", - "id": "TransresnetAgent", + "id": "Speaker 2", "final_rating": 0 } ], diff --git a/tests/crowdsourcing/tasks/model_chat/test_model_chat.py b/tests/crowdsourcing/tasks/model_chat/test_model_chat.py index 8cac9644f24..c5c4806d262 100644 --- a/tests/crowdsourcing/tasks/model_chat/test_model_chat.py +++ b/tests/crowdsourcing/tasks/model_chat/test_model_chat.py @@ -16,7 +16,7 @@ # Inputs -AGENT_DISPLAY_IDS = ('Worker',) +AGENT_DISPLAY_IDS = ('Speaker 1',) AGENT_MESSAGES = [ ("What are you nervous about?",), ("Do you have any plans for the weekend?",), diff --git a/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py b/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py index feb6800535f..34f9c44a00a 100644 --- a/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py +++ b/tests/crowdsourcing/tasks/model_chat/test_model_image_chat.py @@ -23,7 +23,7 @@ # Inputs -AGENT_DISPLAY_IDS = ('Worker',) +AGENT_DISPLAY_IDS = ('Speaker 1',) AGENT_MESSAGES = [ ("Response 1",), ("Response 2",), From 12a33664a1a91573148d6326326948350c475f7a Mon Sep 17 00:00:00 2001 From: EricMichaelSmith Date: Thu, 23 Dec 2021 15:14:22 -0500 Subject: [PATCH 4/5] Fixes --- .../crowdsourcing/tasks/model_chat/utils.py | 17 +++-- .../model_chat/expected_states/state.json | 71 ++++++++++++++----- 2 files changed, 65 insertions(+), 23 deletions(-) diff --git a/parlai/crowdsourcing/tasks/model_chat/utils.py b/parlai/crowdsourcing/tasks/model_chat/utils.py index dd57ebe7a7c..f817ae184a9 100644 --- a/parlai/crowdsourcing/tasks/model_chat/utils.py +++ b/parlai/crowdsourcing/tasks/model_chat/utils.py @@ -381,13 +381,16 @@ def _check_final_chat_data( for actual_message, expected_message in zip( actual_value[key_inner], expected_value_inner ): - self.assertEqual( - {k: v for k, v in actual_message.items() if k != 'message_id'}, - { - k: v - for k, v in expected_message.items() - if k != 'message_id' - }, + clean_actual_message = { + k: v for k, v in actual_message.items() if k != 'message_id' + } + clean_expected_message = { + k: v for k, v in expected_message.items() if k != 'message_id' + } + self.assertDictEqual( + clean_actual_message, + clean_actual_message, + f'The following dictionaries are different: {clean_actual_message} and {clean_expected_message}', ) elif key_inner == 'task_description': for (key_inner2, expected_value_inner2) in expected_value_inner.items(): diff --git a/tests/crowdsourcing/tasks/model_chat/expected_states/state.json b/tests/crowdsourcing/tasks/model_chat/expected_states/state.json index 87f7885c653..80cb3bb76e1 100644 --- a/tests/crowdsourcing/tasks/model_chat/expected_states/state.json +++ b/tests/crowdsourcing/tasks/model_chat/expected_states/state.json @@ -5,7 +5,9 @@ { "data": { "message_id": "3a289317-8519-4da8-be8f-9a515a6ec3da", - "state": {"agent_display_name": "Speaker 1"} + "state": { + "agent_display_name": "Speaker 1" + } }, "packet_type": "update_status", "receiver_id": "1", @@ -19,7 +21,11 @@ "fake_start": true, "id": "Speaker 1", "message_id": "4f72800c-e1ff-4411-8ee5-286336c27859", - "text": "Hi!" + "text": "Hi!", + "task_data": { + "human_persona_string_1": "", + "human_persona_string_2": "" + } }, "packet_type": "agent_action", "receiver_id": "1", @@ -233,9 +239,13 @@ }, { "final_chat_data": { - "acceptability_violations": [null], + "acceptability_violations": [ + null + ], "additional_context": null, - "assignment_ids": ["1"], + "assignment_ids": [ + "1" + ], "bad_workers": [], "context_dataset": null, "dialog": [ @@ -245,7 +255,11 @@ "fake_start": true, "id": "Speaker 1", "message_id": "4f72800c-e1ff-4411-8ee5-286336c27859", - "text": "Hi!" + "text": "Hi!", + "task_data": { + "human_persona_string_1": "", + "human_persona_string_2": "" + } }, { "agent_idx": 1, @@ -352,7 +366,9 @@ "text": "I don't know." } ], - "hit_ids": ["1"], + "hit_ids": [ + "1" + ], "person1_seed_utterance": null, "person2_seed_utterance": null, "personas": null, @@ -368,7 +384,9 @@ "datatype": "train", "image_mode": "raw", "hide_labels": false, - "multitask_weights": [1], + "multitask_weights": [ + 1 + ], "batchsize": 1, "dynamic_batching": null, "verbose": false, @@ -389,7 +407,9 @@ "starttime": "Jan08_11-16" } }, - "workers": ["--NOT-MTURK-AGENT-MOCK_WORKER_0"] + "workers": [ + "--NOT-MTURK-AGENT-MOCK_WORKER_0" + ] } }, { @@ -402,7 +422,11 @@ "id": "Speaker 1", "text": "Hi!", "fake_start": true, - "agent_idx": 0 + "agent_idx": 0, + "task_data": { + "human_persona_string_1": "", + "human_persona_string_2": "" + } }, { "agent_idx": 1, @@ -509,11 +533,19 @@ "final_rating": 4 } ], - "workers": ["--NOT-MTURK-AGENT-MOCK_WORKER_0"], + "workers": [ + "--NOT-MTURK-AGENT-MOCK_WORKER_0" + ], "bad_workers": [], - "acceptability_violations": [null], - "hit_ids": ["1"], - "assignment_ids": ["1"], + "acceptability_violations": [ + null + ], + "hit_ids": [ + "1" + ], + "assignment_ids": [ + "1" + ], "task_description": { "annotations_config": "{\n \"config\": {\n \"bucket_0\": {\n \"name\": \"Bucket 0\",\n \"description\": \"this response implies something...0\"\n },\n \"bucket_1\": {\n \"name\": \"Bucket 1\",\n \"description\": \"this response implies something...1\"\n },\n \"bucket_2\": {\n \"name\": \"Bucket 2\",\n \"description\": \"this response implies something...2\"\n },\n \"bucket_3\": {\n \"name\": \"Bucket 3\",\n \"description\": \"this response implies something...3\"\n },\n \"bucket_4\": {\n \"name\": \"Bucket 4\",\n \"description\": \"this response implies something...4\"\n },\n \"none_all_good\": {\n \"name\": \"None, all good\",\n \"description\": \"This response implies that there are no problems with the data\"\n }\n }\n}\n", "model_nickname": "fixed_response", @@ -527,7 +559,9 @@ "datatype": "train", "image_mode": "raw", "hide_labels": false, - "multitask_weights": [1], + "multitask_weights": [ + 1 + ], "batchsize": 1, "dynamic_batching": null, "verbose": false, @@ -564,7 +598,12 @@ "timestamp": 1606748839.8638673 }, { - "data": {"MEPHISTO_is_submit": true, "task_data": {"final_data": {}}}, + "data": { + "MEPHISTO_is_submit": true, + "task_data": { + "final_data": {} + } + }, "packet_type": "agent_action", "receiver_id": "Mephisto", "sender_id": "1", @@ -572,4 +611,4 @@ } ] } -} +} \ No newline at end of file From 1644ece870d83b43cc8658271c566de513f6b8e1 Mon Sep 17 00:00:00 2001 From: EricMichaelSmith Date: Thu, 23 Dec 2021 17:17:16 -0500 Subject: [PATCH 5/5] Fixes --- parlai/crowdsourcing/tasks/model_chat/utils.py | 2 +- .../tasks/model_chat/expected_states/final_chat_data.json | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/parlai/crowdsourcing/tasks/model_chat/utils.py b/parlai/crowdsourcing/tasks/model_chat/utils.py index f817ae184a9..3a69a4d3655 100644 --- a/parlai/crowdsourcing/tasks/model_chat/utils.py +++ b/parlai/crowdsourcing/tasks/model_chat/utils.py @@ -389,7 +389,7 @@ def _check_final_chat_data( } self.assertDictEqual( clean_actual_message, - clean_actual_message, + clean_expected_message, f'The following dictionaries are different: {clean_actual_message} and {clean_expected_message}', ) elif key_inner == 'task_description': diff --git a/tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data.json b/tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data.json index 0fb70e37045..4f4a5daa6b0 100644 --- a/tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data.json +++ b/tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data.json @@ -11,7 +11,11 @@ "text": "Hi!", "fake_start": true, "agent_idx": 0, - "message_id": "4f72800c-e1ff-4411-8ee5-286336c27859" + "message_id": "4f72800c-e1ff-4411-8ee5-286336c27859", + "task_data": { + "human_persona_string_1": "", + "human_persona_string_2": "" + } }, { "agent_idx": 1,