diff --git a/parlai/crowdsourcing/tasks/model_chat/model_chat_blueprint.py b/parlai/crowdsourcing/tasks/model_chat/model_chat_blueprint.py index fa09d239c1f..8c9a543b7a9 100644 --- a/parlai/crowdsourcing/tasks/model_chat/model_chat_blueprint.py +++ b/parlai/crowdsourcing/tasks/model_chat/model_chat_blueprint.py @@ -157,7 +157,9 @@ class BaseModelChatBlueprintArgs(ParlAIChatBlueprintArgs): ) allowed_worker_qualification: Optional[str] = field( default=None, - metadata="The qualification name for the workers that are exclusively allowed to do the HITs from this task.", + metadata={ + "help": "The qualification name for the workers that are exclusively allowed to do the HITs from this task." + }, ) @@ -203,9 +205,11 @@ def assert_task_args( f'"~" can\'t currently be parsed in the chat data folder path ' f'{args.blueprint.chat_data_folder}' ) - # Currently Hydra overrides the tilde key at lower levels as described here: https://hydra.cc/docs/next/advanced/override_grammar/basic/#grammar - # Thus the TILDE key cannot be used in replacement for $HOME variable - # Some hacky solution can probably be achieved but won't be good code so for now this assert is written as a placeholder + # Currently Hydra overrides the tilde key at lower levels as described here: + # https://hydra.cc/docs/next/advanced/override_grammar/basic/#grammar + # Thus the TILDE key cannot be used in replacement for $HOME variable. + # Some hacky solution can probably be achieved but won't be good code so for now + # this assert is written as a placeholder if args.blueprint.get("annotations_config_path", "") != "": full_path = os.path.expanduser(args.blueprint.annotations_config_path) diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/README.md b/parlai/crowdsourcing/tasks/multi_model_chat/README.md new file mode 100644 index 00000000000..855b8cc1942 --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/README.md @@ -0,0 +1,14 @@ +# Multi-party model chat +An extension of the model chat that runs between the human crowdworker and mutliple model-controlled agents. +```.sh +# assuming you are in multi_model_chat directory +python -m parlai.crowdsourcing.tasks.model_chat.run \ +--config-path "$(eval pwd)/hydra_configs" conf=multiparty task_dir="$(eval pwd)" +``` + +# Modules structure +Most of the components are inherited from the regular model chat and have the same functionalities. + +The main exta piece here is the `agents.py` module which is in charge of creating custom agents for controlling the conversation flow and utterance responses. + +The `ContextGenerator` class, which is part of the worlds, generates location descriptions and personas. There is a minimal implementation of it here with only 2 hard-coded settings. The users must re-implement that in practice. diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/agents.py b/parlai/crowdsourcing/tasks/multi_model_chat/agents.py new file mode 100644 index 00000000000..738bf8360f1 --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/agents.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Optional, Dict +import random + +from parlai.core.agents import Agent +from parlai.core.opt import Opt +from parlai.core.params import ParlaiParser +from parlai.core.agents import create_agent_from_model_file +from parlai.core.loader import load_agent_module +from parlai.utils.logging import logging + +PERSONA_SETTER_AGENT = 'persona-agent' + + +# The delimiter for the parts of the messages (eg, speaker : timestamp : text) +DEFAULT_SPEAKER_TOKEN_DELIM = ':' +SILENCE_TOKEN = '__SILENCE__' + +DECISION_MODEL_OVERRIDES = { + 'interactive_mode': True, + 'skip_generation': False, + 'fp16': True, + 'batchsize': 1, +} + +SPEECH_MODEL_OVERRIDES = { + 'interactive_mode': True, + 'skip_generation': False, + 'fp16': True, + 'batchsize': 1, + 'inference': 'beam', + 'beam_size': 3, + 'beam_min_length': 20, + 'beam_block_ngram': 3, + 'beam_context_block_ngram': 3, +} + + +def flatten_personas(personas: Dict, delim='\n', bb3_format=False): + personass_str_parts = [] + if bb3_format: + for i, p in enumerate(personas): + personass_str_parts.append(f"Person {i+1} is: {p['name']}") + personass_str_parts.append(f"Person {i+1}'s Persona: {p['persona']}") + else: + personass_str_parts.append('__personas__') + personass_str_parts.extend([f"{p['name']}: {p['persona']}" for p in personas]) + personass_str_parts.append('__end-personas__') + + return delim.join(personass_str_parts) + + +def flatten_location(location: Dict, delim='\n', bb3_format=False): + if bb3_format: + location_str_parts = [ + f'Setting: {location["name"]}', + f'Description: {location["description"]}', + ] + else: + location_str_parts = [ + '__location__', + f"{location['name']}: {location['description']}", + '__end-location__', + ] + + return delim.join(location_str_parts) + + +class RandomSpeakerDecicionsAgent(Agent): + """ + Randomly decides who speaks next. + """ + + def set_characters(self, characters): + self.characters = characters + + def act(self): + assert hasattr(self, 'characters'), 'Personas are not set.' + return { + 'text': random.choice(self.characters), + 'id': 'RandomOrderDecision', + 'episode_done': False, + } + + +class MultipartyModelChatAgent(Agent): + """ + Agent to use in a live chat with human. + + The assumption is that there is only 1 human; all other characters are handled by + model. We still use the regular observe and act cycles of any other ParlAI agent, + But after each observe, model decides whose turn is next and if it is the human + character's turn it responds with silence. Otherwise it uses its utterance + generation model and generates a text response. + """ + + def __init__(self, opt: Opt, shared=None): + self.id = 'MultipartyChatAgent' + self.history = [] + self.utterance_delimiter = opt['utterance_delimiter'] + self.include_speaker_in_context = opt['include_speaker_in_context'] + self.add_speaker_to_context_end = opt['add_speaker_to_context_end'] + self.speaker_token_delimiter = opt['speaker_token_delimiter'] + self.add_personas_to_context = opt['add_personas_to_context'] + self.add_location_to_context = opt['add_location_to_context'] + self.context_format = opt['context_format'] + + if not shared: + self._decision_agent = self._create_decision_agent(opt) + self._speech_agent = self._create_speech_agent(opt) + else: + self._decision_agent = shared['decision_agent'].clone() + self._speech_agent = shared['speech_agent'].clone() + + super().__init__(opt, shared) + + def share(self): + """ + Share model parameters. + """ + shared = super().share() + shared['decision_agent'] = self._decision_agent + shared['speech_agent'] = self._speech_agent + return shared + + @classmethod + def add_cmdline_args( + cls, parser: ParlaiParser, partial_opt: Optional[Opt] = None + ) -> ParlaiParser: + agent = parser.add_argument_group( + 'Multiparty agent for model chat (human evals).' + ) + agent.add_argument( + '--decision-agent', + type=str, + help='Agent for deciding the next speaker.', + ) + agent.add_argument( + '--decision-model-file', + type=str, + help='Model file for deciding the next speaker (will be ignored if used with --decision-agent).', + ) + agent.add_argument( + '--speech-agent', + type=str, + help='Agent for generating the response text.', + ) + agent.add_argument( + '--speech-model-file', + type=str, + help='Model file for generating the response text (will be ignored if used with --speech-agent).', + ) + agent.add_argument( + '--utterance-delimiter', + type=str, + default='\n', + help="A string used to separate each utterance in the context. Defaults to newline. For example, 'A: Hello\nB: Hi there'.", + ) + agent.add_argument( + '--include-speaker-in-context', + type='bool', + default=True, + help="Whether to include speaker labels in the context. " + "For example, message = { text: 'Rachel: Hi' } instead of message = { text: 'Hi' }", + ) + agent.add_argument( + '--add-speaker-to-context-end', + type='bool', + default=True, + help='Append the current speaker to the end of each context.', + ) + agent.add_argument( + '--speaker-token-delimiter', + type=str, + default=DEFAULT_SPEAKER_TOKEN_DELIM, + help="The token to use to separate the speaker label from the actual utterance in `obs['text']`.", + ) + agent.add_argument( + '--add-personas-to-context', + type=bool, + default=True, + help="If true, will add the flattened personas to the contet end.", + ) + agent.add_argument( + '--add-location-to-context', + type=bool, + default=True, + help="If true, will add the flattened location to the contet end.", + ) + agent.add_argument( + '--context-format', + type=str, + default='multilight', + choices=('bb3', 'multilight', 'light'), + help="The token to use to separate the speaker label from the actual utterance in `obs['text']`.", + ) + return parser + + def _create_decision_agent(self, opt): + logging.info('Creating the decision agent.') + if opt.get('decision_agent'): + m = load_agent_module(opt['decision_agent']) + return m(opt) + elif 'decision_model_file' in opt: + return create_agent_from_model_file( + opt['decision_model_file'], opt_overrides=DECISION_MODEL_OVERRIDES + ) + else: + raise ValueError( + "The opt must have 'decision_agent' or 'decision_model_file'." + ) + + def _create_speech_agent(self, opt): + logging.info('Creating the speech agent.') + if opt.get('speech_agent'): + return load_agent_module(opt['speech_agent']) + elif 'speech_model_file' in opt: + return create_agent_from_model_file( + opt['speech_model_file'], opt_overrides=SPEECH_MODEL_OVERRIDES + ) + else: + raise ValueError("The opt must have 'speech_agent' or 'speech_model_file'.") + + def get_context(self, context_format=None): + """ + Generates the text that goes into each of the models (speaker decision and the + speech). + """ + if not context_format: + context_format = self.context_format + + context_parts = [] + if context_format in ('bb3', 'multilight'): + use_bb3_format = context_format == 'bb3' + if self.add_location_to_context: + context_parts.append( + flatten_location(self.location, bb3_format=use_bb3_format) + ) + if self.add_personas_to_context: + context_parts.append( + flatten_personas(self.personas, bb3_format=use_bb3_format) + ) + elif context_format == 'light': + context_parts.append('_task_speech') + if self.add_location_to_context: + context_parts.append(f'_setting_name {self.location["name"]}') + context_parts.append(f'_setting_desc {self.location["description"]}') + if self.add_personas_to_context: + context_parts.append(f'_self_name {self.personas[0]["name"]}') + context_parts.append(f'_self_persona {self.personas[0]["persona"]}') + else: + raise ValueError( + f'The requested context format ("{self.context_format}") is not implemented yet.' + ) + + context_parts.extend(self.history) + return self.utterance_delimiter.join(context_parts) + + def update_history(self, act): + utterance_line = act["text"] + if self.include_speaker_in_context: + utterance_line = f'{act["id"]}: {utterance_line}' + self.history.append(utterance_line) + + def get_speaker_index(self, spk): + assert hasattr(self, 'characters'), 'Personas are not set.' + return self.characters.index(spk) + + def is_human_turn(self, turn_act): + # The assumption here is that the character with index 0 is the human. + spk = turn_act['text'].lower() + return spk not in self.characters or self.get_speaker_index(spk) == 0 + + def is_bot_turn(self, turn_act): + not self.is_human_turn(turn_act) + + def observe(self, observation): + if observation['id'] == PERSONA_SETTER_AGENT: + self.location = observation['location'] + self.personas = observation['personas'] + self.characters = [p['name'].lower() for p in self.personas] + if hasattr(self._decision_agent, 'set_characters'): + # The random agent has this. + self._decision_agent.set_characters(self.characters) + return + + observation['id'] = self.personas[0]['name'] + self.update_history(observation) + + def get_next_turn(self): + context = self.get_context(context_format=self.context_format) + logging.debug(f'The decision model context:{context}') + self._decision_agent.observe({'text': context, 'episode_done': False}) + next_turn = self._decision_agent.act() + self._decision_agent.reset() + return next_turn + + def act(self): + next_turn = self.get_next_turn() + speaker = next_turn["text"] + logging.info(f'The next round assigned to {speaker}') + if self.is_human_turn(next_turn): + # Returning empty for passing the turn to the human. + return {'text': '', 'episode_done': False, 'human_turn': True} + else: + context = self.get_context() + if self.add_speaker_to_context_end: + context = self.utterance_delimiter.join( + [context, f'{speaker}{self.speaker_token_delimiter}'] + ) + logging.debug(f'The speech model context:\n{context}') + self._speech_agent.observe({'text': context, 'episode_done': False}) + response = self._speech_agent.act() + self._speech_agent.reset() + response.force_set('id', speaker) + self.update_history(response) + return response diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/compile_results.py b/parlai/crowdsourcing/tasks/multi_model_chat/compile_results.py new file mode 100644 index 00000000000..04e13758644 --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/compile_results.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python3 + +# Copyright (c) Facebook, Inc. and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import numpy as np +import pandas as pd + +from parlai.crowdsourcing.tasks.model_chat.analysis.compile_results import ( + ModelChatResultsCompiler, +) + + +class MultiLIGHTModelChatResultsCompiler(ModelChatResultsCompiler): + """ + Compile and save results of human+model chats, based on MultiLIGHT model chats. + + Results will be saved on the level of specific conversations, as well as aggregated + up the level of each worker as a whole. + """ + + def compile_results(self) -> pd.DataFrame: + task_units_data = self.get_task_data() + # Read in each file + num_convos_with_no_save_data = 0 + num_wrong_status_convos = 0 + num_complete_convos = 0 + complete_convos_per_model = {} + bad_conversations = [] + stat_counts = {} + worker_stats = {} + worker_conversation_counts = {} + total_utterances = 0 + + conversation_idx = 0 + conversation_dfs = [] + for task_unit in task_units_data: + + worker_id = task_unit['worker_id'] + assignment_id = task_unit['assignment_id'] + + # Only include the first max_convos_per_worker conversations from a + # worker to avoid biasing + if worker_id in worker_conversation_counts: + conversations_so_far = worker_conversation_counts[worker_id] + else: + conversations_so_far = 0 + worker_conversation_counts[worker_id] = conversations_so_far + 1 + if ( + self.max_convos_per_worker != -1 + and conversations_so_far >= self.max_convos_per_worker + ): + print( + f'Had {conversations_so_far} conversation(s) already from this worker {worker_id}. Skipping {assignment_id}.' + ) + continue + + persona_setting_message = task_unit['data']['messages'][2] + personas = persona_setting_message['task_data']['personas'] + # The task always assigns the first persona to the human. + human_speaker_character = personas[0]['name'] + + saved_data = task_unit['data']['save_data']['custom_data'] + conv_data = saved_data['dialog'] + + # Check if need to block the turker + word_counts = [ + len(d['text'].split(' ')) + for d in conv_data + if d['id'] == human_speaker_character + ] + human_utterances = [ + d['text'] for d in conv_data if d['id'] == human_speaker_character + ] + + if np.average(word_counts) < self.min_word_count: + bad_conversations.append(saved_data) + print( + f'Bad complete conversation, words from human: {human_utterances}. Skipping.' + ) + continue + + model_nickname = saved_data['task_description']['model_nickname'] + if model_nickname not in stat_counts: + stat_counts[model_nickname] = {} + + if 'max_turn_rate' not in stat_counts[model_nickname]: + stat_counts[model_nickname]['max_turn_rate'] = [] + + if 'min_turn_rate' not in stat_counts[model_nickname]: + stat_counts[model_nickname]['min_turn_rate'] = [] + + if model_nickname in complete_convos_per_model: + complete_convos_per_model[model_nickname] += 1 + else: + complete_convos_per_model[model_nickname] = 1 + + # Extract non-message info + info_dict = { + 'worker': worker_id, + 'model_nickname': model_nickname, + 'bad_workers': ','.join(saved_data['bad_workers']), + 'hit_id': saved_data['hit_ids'][0], + 'assignment_id': assignment_id, + 'context_dataset': saved_data['context_dataset'], + 'additional_context': saved_data['additional_context'], + } + + info_dict[ + 'acceptability_violations_0' + ] = self.acceptability_checker.check_messages( + messages=human_utterances, + is_worker_0=True, + violation_types=self.acceptability_checker.ALL_VIOLATION_TYPES, + ) + + # Compile personas and previous utterances + df = pd.DataFrame( + [], + columns=[ + 'worker_id', + 'hit_id', + 'model_nickname', + 'conversation_idx', + 'turn_idx', + 'agent_idx', + 'text', + ] + + self.problem_buckets, + ) + text_parts = [] + for p in personas: + text_parts.append(f'{p["name"]}: {p["persona"]}') + + new_row = pd.DataFrame( + { + 'worker_id': info_dict['worker'], + 'hit_id': info_dict['hit_id'], + 'model_nickname': model_nickname, + 'conversation_idx': conversation_idx, + 'turn_idx': -1, + 'agent_idx': 1, + 'text': '\n'.join(text_parts), + **{bucket: '' for bucket in self.problem_buckets}, + }, + index=[0], + ) + df = pd.concat( + [df, new_row], + ignore_index=True, + ) + + total_utterances += len( + [d for d in saved_data["dialog"] if d['id'] == human_speaker_character] + ) + if len(saved_data['dialog']) > 20: + print( + f'Got long dialogue of {len(saved_data["dialog"])} utterances, hit id:' + f' {info_dict["hit_id"]}, model_nickname: {model_nickname}.' + ) + + speaker_count = dict() + for p in personas: + speaker_count[p['name'].lower()] = 0 + for utterance_idx, utt in enumerate(saved_data['dialog']): + + d = { + 'worker_id': info_dict['worker'], + 'hit_id': info_dict['hit_id'], + 'model_nickname': model_nickname, + 'conversation_idx': conversation_idx, + 'turn_idx': utterance_idx, + 'agent_idx': utt['agent_idx'], + 'text': utt['text'], + **{bucket: '' for bucket in self.problem_buckets}, + } + speaker_count[utt['id'].lower()] += 1 + + if utt['agent_idx'] == 1: + + d['final_rating'] = utt.get('final_rating') + + if self.use_problem_buckets: + if 'problem_data' not in utt: + for bucket in self.problem_buckets: + d[bucket] = 'MALFORMED' + print( + f'Warning got MALFORMED utterance problem data inside complete convo: {utt}. Skipping.' + ) + continue + else: + for bucket in self.regular_buckets + ['none']: + d[bucket] = utt['problem_data'][bucket] + for k in self.regular_buckets + ['none']: + if k not in stat_counts[model_nickname]: + stat_counts[model_nickname][k] = [] + stat_counts[model_nickname][k].append(d[k]) + + if 'total' not in stat_counts[model_nickname]: + stat_counts[model_nickname]['total'] = 0 + stat_counts[model_nickname]['total'] += 1 + if d['final_rating'] is not None: + # Only one the last utterance (agent idx == 1) + if 'count_ratings' not in stat_counts[model_nickname]: + stat_counts[model_nickname]['count_ratings'] = 0 + stat_counts[model_nickname]['count_ratings'] += 1 + if 'ratings' not in stat_counts[model_nickname]: + stat_counts[model_nickname]['ratings'] = [] + stat_counts[model_nickname]['ratings'].append( + int(d['final_rating']) + ) + + else: + + # Counting some aspects of the human's utterances + if 'human_utterance_count' not in stat_counts[model_nickname]: + stat_counts[model_nickname]['human_utterance_count'] = 0 + stat_counts[model_nickname]['human_utterance_count'] += 1 + + if 'human_word_count' not in stat_counts[model_nickname]: + stat_counts[model_nickname]['human_word_count'] = 0 + stat_counts[model_nickname]['human_word_count'] += len( + d['text'].strip().split(' ') + ) + + if 'human_question_count' not in stat_counts[model_nickname]: + stat_counts[model_nickname]['human_question_count'] = 0 + stat_counts[model_nickname]['human_question_count'] += d[ + 'text' + ].count('?') + + df = pd.concat([df, pd.DataFrame(d, index=[0])], ignore_index=True) + + if info_dict['worker'] not in worker_stats: + worker_stats[info_dict['worker']] = {'conversations': 0} + worker_stats[info_dict['worker']]['conversations'] += 1 + + # Logic for calculating percent of conversations that are clean + if 'count_convos' not in stat_counts[model_nickname]: + stat_counts[model_nickname]['count_convos'] = 0 + stat_counts[model_nickname]['count_convos'] += 1 + + stat_counts[model_nickname]['max_turn_rate'].append( + max(speaker_count.values()) + ) + stat_counts[model_nickname]['min_turn_rate'].append( + min(speaker_count.values()) + ) + + # Adding the full conversation to the list of conversations + conversation_dfs.append(df) + conversation_idx += 1 + + for m, conversation_count in complete_convos_per_model.items(): + print(f'Got {conversation_count} complete conversation(s) for model: {m}') + + print(f'{num_complete_convos:d} complete conversation(s) collected.') + print(f'{len(bad_conversations):d} bad conversation(s).') + num_approved_convos = num_complete_convos - len(bad_conversations) + print(f'{num_approved_convos:d} approved conversation(s).') + print(f'({num_wrong_status_convos:d} wrong status conversation(s) collected.)') + print( + f'({num_convos_with_no_save_data:d} conversation(s) collected with no saved data.)' + ) + for model_nickname, model_stats_dict in stat_counts.items(): + print(f'---{model_nickname}---') + for p, v in model_stats_dict.items(): + if p == 'count_ratings': + continue + if p == 'ratings': + print( + f'Average Engaging-ness Rating: {np.average(model_stats_dict["ratings"])}' + f' ({model_stats_dict["count_ratings"]} ratings)' + ) + print( + f'Engaging-ness Rating Variance: {np.std(model_stats_dict["ratings"])}' + f' ({model_stats_dict["count_ratings"]} ratings)' + ) + elif p == 'human_word_count' or p == 'human_question_count': + print( + f'{p}: {v} ({v/model_stats_dict["human_utterance_count"]:.3})' + ) + elif p == 'human_utterance_count': + print(f'{p}: {v}') + elif p == 'count_convos': + print(f'{p}: {v}') + elif p in ('min_turn_rate', 'max_turn_rate'): + print(f'{p}: {np.average(v)}') + elif self.use_problem_buckets and p == 'convo_clean': + print(f'{p}: {v} ({v/model_stats_dict["count_convos"]:.2%})') + else: + if p == 'total': + print(f'{p}: {v/model_stats_dict["total"]:.2%}') + else: + print(f'[DEBUG numpy] AVG {p}: {np.average(v):.2%}') + print(f'[DEBUG numpy] STD {p}: {np.std(v):.2%}') + print(f'{p}: {sum(v)/model_stats_dict["total"]:.2%}') + + print('Printing worker IDs not already in block list to add...') + for b in bad_conversations: + worker_id = b['workers'][0] + if worker_id not in self.worker_block_list: + print(f"""'{worker_id}',""") + print('Done printing bad workers.') + + # Save full results + all_conversations_df = pd.DataFrame() + for df in conversation_dfs: + all_conversations_df = pd.concat([all_conversations_df, df]) + print(f'\nWorker conversation counts: {worker_conversation_counts}') + + return all_conversations_df + + +if __name__ == '__main__': + parser_ = MultiLIGHTModelChatResultsCompiler.setup_args() + args_ = parser_.parse_args() + MultiLIGHTModelChatResultsCompiler(vars(args_)).compile_and_save_results() diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/chat_app_with_onboarding.jsx b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/chat_app_with_onboarding.jsx new file mode 100644 index 00000000000..604bb9ec004 --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/chat_app_with_onboarding.jsx @@ -0,0 +1,280 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * All rights reserved. + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +// Copies code directly from `bootstrap-chat` in order to make a chat app +// that renders with a different frontend onboarding flow. + +import React from "react"; + +import { + MephistoContext, + useMephistoLiveTask, + AGENT_STATUS, + STATUS_TO_TEXT_MAP, +} from "mephisto-task"; +import { BaseFrontend, AppContext } from "bootstrap-chat"; +import { OnboardingComponent } from "./onboarding_components.jsx" + +/* ================= Application Components ================= */ + +const INPUT_MODE = { + WAITING: "waiting", + INACTIVE: "inactive", + DONE: "done", + READY_FOR_INPUT: "ready_for_input", +}; + +function CustomOnboardingChatApp({ + renderMessage, + renderSidePane, + renderTextResponse, + renderResponse, + onMessagesChange, + propAppSettings={}, +}) { + const [taskContext, updateContext] = React.useReducer( + (oldContext, newContext) => { + return { ...oldContext, ...newContext }; + }, + {currentAgentNames: []} + ); + + const [messages, addMessage] = React.useReducer( + (previousMessages, newMessage) => { + // we clear messages by sending false + return newMessage === false ? [] : [...previousMessages, newMessage]; + }, + [] + ); + + const initialAppSettings = { + volume: 1, + isReview: false, + isCoverPage: false, + numMessages: 0, + numTurns: 0, + useTurns: true, + ...propAppSettings + }; + const [appSettings, setAppSettings] = React.useReducer( + (prevSettings, newSettings) => Object.assign({}, prevSettings, newSettings), + initialAppSettings + ); + const [inputMode, setInputMode] = React.useState(INPUT_MODE.WAITING); + + React.useEffect(() => { + if (onMessagesChange) { + onMessagesChange(messages); + } + let n = 0; + for (let i=0 ; i 0) { + updateContext(remainingState); + } + } + + let mephistoProps = useMephistoLiveTask({ + onStatusUpdate: ({ status }) => { + if ( + [ + AGENT_STATUS.DISCONNECT, + AGENT_STATUS.RETURNED, + AGENT_STATUS.EXPIRED, + AGENT_STATUS.TIMEOUT, + AGENT_STATUS.PARTNER_DISCONNECT, + AGENT_STATUS.MEPHISTO_DISCONNECT, + ].includes(status) + ) { + setInputMode(INPUT_MODE.INACTIVE); + updateContext({ + doneText: STATUS_TO_TEXT_MAP[status], + task_done: status == AGENT_STATUS.PARTNER_DISCONNECT, + }); + } + }, + onLiveUpdate: (message) => { + console.log("Live message", message) + if (message.task_data !== undefined) { + handleStateUpdate(message.task_data); + } + if (message.text !== undefined) { + addMessage(message); + } + + // For handling reconnected packets and properly updating state + // during turns. + if ( + taskContext.currentAgentNames && + message.id in taskContext.currentAgentNames && + appSettings.useTurns + ) { + // This was our own message, so update to not requesting + handleStateUpdate({ live_update_requested: false }); + } + }, + }); + + let { + blockedReason, + blockedExplanation, + taskConfig, + isPreview, + previewHtml, + isLoading, + agentId, + handleSubmit, + connect, + destroy, + sendLiveUpdate, + isOnboarding, + agentStatus, + } = mephistoProps; + + React.useEffect(() => { + if (agentId) { + console.log("connecting..."); + connect(agentId); + } + }, [agentId]); + + React.useEffect(() => { + if (isOnboarding && agentStatus === AGENT_STATUS.WAITING) { + handleSubmit(); + } + }, [isOnboarding, agentStatus]); + + React.useEffect(() => { + // clear messages when onboarding changes status + addMessage(false); + }, [isOnboarding]) + + const handleMessageSend = React.useCallback( + (message) => { + message = { + ...message, + id: agentId, + episode_done: taskContext?.task_done || false, + }; + return sendLiveUpdate(message) + .then(addMessage) + .then(() => { + if (appSettings.useTurns) { + handleStateUpdate({ live_update_requested: false }); + } + }); + }, + [agentId, taskContext?.task_done, addMessage, setInputMode] + ); + + if (blockedReason !== null) { + return

{blockedExplanation}

; + } + if (isLoading) { + return
Initializing...
; + } + if (isPreview) { + if (!taskConfig.has_preview) { + return ; + } + if (previewHtml === null) { + return
Loading...
; + } + return
; + } + if (isOnboarding) { + return { + handleMessageSend({text: '', task_data: dat}); + }} + />; + } + + return ( + + { + destroy(); + handleSubmit({}); + }, + }} + > +
+ +
+ + + ); +} + +function TaskPreviewView({ description }) { + return ( +
+
+
+ ); +} + +export { CustomOnboardingChatApp, AppContext }; \ No newline at end of file diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/checkboxes.jsx b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/checkboxes.jsx new file mode 100644 index 00000000000..3deda1474b3 --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/checkboxes.jsx @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * All rights reserved. + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +import React from "react"; + +function Checkboxes({ + annotationBuckets, + turnIdx, + askReason, + annotations, + onUpdateAnnotations, + enabled=true, +}) { + var reasonComponent = ( +
+

+
+
Why did you select the checkboxes you did?
+ +
+
+ ) + if (!askReason) { + reasonComponent = ''; + } + // TODO: add support for radio input type + let input_type = "checkbox"; + const showLineBreaks = annotationBuckets.hasOwnProperty("show_line_breaks") ? annotationBuckets.show_line_breaks : false; + const numBuckets = Object.keys(annotationBuckets.config).length; + return ( +
+ { + Object.keys(annotationBuckets.config).map((c, checkboxIdx) => ( + <> + + { + let newVal = evt.target.checked; + let oldAnnotations = Object.assign({}, annotations); + oldAnnotations[c] = newVal; + onUpdateAnnotations(oldAnnotations); + }} + disabled={!enabled} + /> + + {annotationBuckets.config[c].name} + + + {(showLineBreaks && checkboxIdx < numBuckets - 1) ?

: ''} + + )) + } +
+ {reasonComponent} +
+ ) +} +// showLineBreaks: show a line break after every checkbox other than the final one + +export { Checkboxes }; diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/error_boundary.jsx b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/error_boundary.jsx new file mode 100644 index 00000000000..df2b0df3986 --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/error_boundary.jsx @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * All rights reserved. + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +import React from "react"; + +class ErrorBoundary extends React.Component { + constructor(props) { + super(props); + this.state = { hasError: false }; + } + + static getDerivedStateFromError(error) { + // Update state so the next render will show the fallback UI. + return { hasError: true }; + } + + componentDidCatch(error, errorInfo) { + // You can also log the error to an error reporting service + console.log(error, errorInfo); + } + + render() { + if (this.state.hasError) { + // You can render any custom fallback UI + return

Something went wrong.

; + } + + return this.props.children; + } + } + +export { ErrorBoundary } \ No newline at end of file diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/message.jsx b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/message.jsx new file mode 100644 index 00000000000..a62585a9e28 --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/message.jsx @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * All rights reserved. + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +import React from "react"; + +import { Checkboxes } from './checkboxes.jsx'; + +function MaybeCheckboxChatMessage({ isSelf, duration, agentName, message = "", checkbox = null }) { + const floatToSide = isSelf ? "right" : "left"; + const alertStyle = isSelf ? "alert-info" : "alert-warning"; + + return ( +
+
+ + {agentName}: + + {checkbox} +
+
+ ); +} + +function RenderChatMessage({ message, mephistoContext, appContext, idx }) { + const { agentId, taskConfig } = mephistoContext; + const { currentAgentNames } = appContext.taskContext; + const { personas } = appContext.taskContext; + const { appSettings, setAppSettings } = appContext; + const { checkboxValues } = appSettings; + const isHuman = (message.id === agentId || message.id == currentAgentNames[agentId]); + const annotationBuckets = taskConfig.annotation_buckets; + const annotationIntro = taskConfig.annotation_question; + + var checkboxes = null; + if (!isHuman && annotationBuckets !== null) { + let thisBoxAnnotations = checkboxValues[idx]; + if (!thisBoxAnnotations) { + thisBoxAnnotations = Object.fromEntries( + Object.keys(annotationBuckets.config).map(bucket => [bucket, false]) + ) + } + checkboxes =
+
+ {annotationIntro} +
+ { + checkboxValues[idx] = newAnnotations; + setAppSettings({checkboxValues}); + } + } + annotationBuckets={annotationBuckets} + turnIdx={idx} + askReason={false} + enabled={idx == appSettings.numMessages - 1} + /> +
; + } + return ( + (message?.text && message.text !== "") ? + : null + ); +} + +export { RenderChatMessage, MaybeCheckboxChatMessage }; \ No newline at end of file diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/onboarding_components.jsx b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/onboarding_components.jsx new file mode 100644 index 00000000000..417966c2a20 --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/onboarding_components.jsx @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * All rights reserved. + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +import React from "react"; +import { ErrorBoundary } from './error_boundary.jsx'; +import { Checkboxes } from './checkboxes.jsx'; +const DEFAULT_MIN_CORRECT = 4; +const DEFAULT_MAX_INCORRECT = 3; +const DEFAULT_MAX_FAILURES_ALLOWED = 1; +var onboardingFailuresCount = 0; + +var renderOnboardingFail = function () { + // Update the UI + document.getElementById("onboarding-submit-button").style.display = 'none'; + + alert('Sorry, you\'ve exceeded the maximum amount of tries to label the sample conversation correctly, and thus we don\'t believe you can complete the task correctly. Please return the HIT.') +} + +function arraysEqual(_arr1, _arr2) { + if (!Array.isArray(_arr1) || ! Array.isArray(_arr2) || _arr1.length !== _arr2.length) + return false; + + var arr1 = _arr1.concat().sort(); + var arr2 = _arr2.concat().sort(); + for (var i = 0; i < arr1.length; i++) { + if (arr1[i] !== arr2[i]) + return false; + } + return true; +} + +var handleOnboardingSubmit = function ({ onboardingData, currentTurnAnnotations, onSubmit }) { + console.log('handleOnboardingSubmit'); + var countCorrect = 0; + var countIncorrect = 0; + for (var turnIdx = 0; turnIdx < onboardingData.dialog.length; turnIdx++) { + var modelUtteranceForTurn = onboardingData.dialog[turnIdx][1]; + var answersForTurn = modelUtteranceForTurn.answers; + if (!answersForTurn) { + continue + } else { + let givenAnswers = currentTurnAnnotations[turnIdx]; + let answerArray = []; + for (let arrayKey in givenAnswers) { + if (givenAnswers[arrayKey]) { + answerArray.push(arrayKey); + } + } + if (arraysEqual(answerArray, answersForTurn)) { + countCorrect += 1; + } else { + countIncorrect += 1; + } + } + } + console.log('correct: ' + countCorrect + ', incorrect: ' + countIncorrect); + const min_correct = onboardingData.hasOwnProperty("min_correct") ? onboardingData.min_correct : DEFAULT_MIN_CORRECT; + const max_incorrect = onboardingData.hasOwnProperty("max_incorrect") ? onboardingData.max_incorrect : DEFAULT_MAX_INCORRECT; + const max_failures_allowed = onboardingData.hasOwnProperty("max_failures_allowed") ? onboardingData.max_failures_allowed : DEFAULT_MAX_FAILURES_ALLOWED; + if (countCorrect >= min_correct && countIncorrect <= max_incorrect) { + onSubmit({ annotations: currentTurnAnnotations, success: true }); + } else { + if (onboardingFailuresCount < max_failures_allowed) { + onboardingFailuresCount += 1; + alert('You did not label the sample conversation well enough. Please try one more time!'); + } else { + renderOnboardingFail(); + onSubmit({ annotations: currentTurnAnnotations, success: false }) + } + } +} + +function OnboardingDirections({ children }) { + return ( +
+
+ {children} +
+
+ ); +} + +function OnboardingUtterance({ + annotationBuckets, + annotationQuestion, + turnIdx, + text, + annotations = null, + onUpdateAnnotation = null, +}) { + var extraElements = ''; + if (turnIdx % 2 == 1) { + extraElements = ''; + extraElements = (

+
+ + + ) + } + return (text !== "") ? ( +
+ {turnIdx % 2 == 0 ? 'YOU' : 'THEM'}: {text} + + {extraElements} + + +
+ ) : null; +} + +function OnboardingComponent({ onboardingData, annotationBuckets, annotationQuestion, onSubmit }) { + if (onboardingData === null) { + return ( +
+ Please wait while we set up the task... +
+ ); + } else { + const [currentTurnAnnotations, setCurrentAnnotations] = React.useState( + Array.from(Array(onboardingData.dialog.length), () => Object.fromEntries( + Object.keys(annotationBuckets.config).map(bucket => [bucket, false])) + ) + ); + return ( +
+ +

Task Description

+
+ To first learn about the labeling task, please evaluate the "THEM" speaker in the conversation below, choosing the correct checkboxes.
+ +
+ +
+ { + onboardingData.dialog.map((turn, idx) => ( +
+ + { + let updatedAnnotations = currentTurnAnnotations.slice() + updatedAnnotations[idx] = newAnnotations; + setCurrentAnnotations(updatedAnnotations); + } + } + /> +
+ )) + } +
+ +
+
+
+
+ +
+
+ ); + } +} + +export { OnboardingComponent, OnboardingUtterance }; \ No newline at end of file diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/response_panes.jsx b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/response_panes.jsx new file mode 100644 index 00000000000..1a4be52c77e --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/response_panes.jsx @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * All rights reserved. + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +import React from "react"; + +import { Button, Col, ControlLabel, Form, FormControl, FormGroup } from "react-bootstrap"; + + +function hasAnyAnnotations(annotations) { + if (!annotations) { + return false; + } + for (const key in annotations) { + if (annotations[key] === true) { + return true; + } + } + return false; +} + +function RatingSelector({ active, ratings, sending, ratingQuestion, ratingIndex, setRatings }) { + const ratingOptions = [ + ); + }) + ); + + function handleRatingSelection(val) { + const newRatings = ratings.map((item, index) => { + if (index === ratingIndex) { + return val; + } else { + return item; + } + }); + setRatings(newRatings); + } + + return ( + + + {ratingQuestion} + + + handleRatingSelection(e.target.value)} + disabled={!active || sending} + > + {ratingOptions} + + + + ); +} + +function FinalSurvey({ taskConfig, onMessageSend, active, currentCheckboxes }) { + const [sending, setSending] = React.useState(false); + + // Set up multiple response questions + let ratingQuestions = taskConfig.final_rating_question.split("|"); + let initialRatings = []; + for (let _ of ratingQuestions) { + initialRatings.push(""); + } + const [ratings, setRatings] = React.useState(initialRatings) + + const tryMessageSend = React.useCallback(() => { + + let all_ratings_filled = ratings.every((r) => r !== ""); + let rating = ratings.join('|'); + + if (all_ratings_filled && active && !sending) { + setSending(true); + onMessageSend({ + text: "", + task_data: { + problem_data_for_prior_message: currentCheckboxes, + final_rating: rating, + }, + }).then(() => { + setSending(false); + }); + } + }, [active, sending, ratings, onMessageSend]); + + const listRatingSelectors = ratingQuestions.map((ratingQuestion, ratingIndex) => { + return ( + + + ); + }); + + if (listRatingSelectors.length > 1) { + // Show ratings to the right of the questions + return ( +
+
+ You've completed the conversation. Please annotate the final turn, fill out + the following, and hit Done. +
+
+
+ {listRatingSelectors} + +
+
+ ); + } else { + // Show the single rating below the single question + return ( +
+
+ You've completed the conversation. Please annotate the final turn, fill out + the following, and hit Done. +
+
+
+ {listRatingSelectors} + +
+
+ ); + } +} + +function CheckboxTextResponse({ onMessageSend, activeText, activeRatingOnly, currentCheckboxes }) { + const [textValue, setTextValue] = React.useState(""); + const [sending, setSending] = React.useState(false); + + const inputRef = React.useRef(); + + React.useEffect(() => { + if (activeText && inputRef.current && inputRef.current.focus) { + inputRef.current.focus(); + } + }, [activeText]); + + const checkedResponses = hasAnyAnnotations(currentCheckboxes); + + const tryMessageSend = React.useCallback(() => { + console.log("Trying to send ... "); + if (((activeRatingOnly && checkedResponses) || (activeText && textValue !== "")) && !sending) { + console.log("Sending ... "); + setSending(true); + onMessageSend({ + text: textValue, + task_data: { problem_data_for_prior_message: currentCheckboxes } + }).then(() => { + setTextValue(""); + setSending(false); + }); + } + }, [activeRatingOnly, checkedResponses, textValue, activeText, sending, onMessageSend]); + + const handleKeyPress = React.useCallback( + (e) => { + if (e.key === "Enter") { + tryMessageSend(); + e.stopPropagation(); + e.nativeEvent.stopImmediatePropagation(); + } + }, + [tryMessageSend] + ); + + const responsePanel = (activeRatingOnly) ? +
+ +
+ : +
+ { + inputRef.current = ref; + }} + value={textValue} + placeholder="Please enter here..." + onKeyPress={(e) => handleKeyPress(e)} + onChange={(e) => setTextValue(e.target.value)} + disabled={!activeText || sending} + /> + +
; + + return ( +
+ {responsePanel} +
+ ); +} + +function ResponseComponent({ taskConfig, appSettings, onMessageSend, activeText, activeRating }) { + + const lastMessageIdx = appSettings.numMessages - 1; + const lastMessageAnnotations = appSettings.checkboxValues[lastMessageIdx]; + + const computedRatingAcive = ( + taskConfig.annotation_buckets === null || activeRating + ) + + // Last message maynot be a "turn" message (one with human or bot utterance). + const turnsFinished = appSettings.numTurns >= taskConfig.min_num_turns; + + const computedTextActive = ( + taskConfig.annotation_buckets === null || + !activeRating && activeText + ); + + if (turnsFinished) { + return ( + + ); + } else { + return ( + + ); + } +} + +export { ResponseComponent }; \ No newline at end of file diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/sidepane.jsx b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/sidepane.jsx new file mode 100644 index 00000000000..da44f0ee19e --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/sidepane.jsx @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * All rights reserved. + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ +import React from "react"; +import "./styles.css"; + +export function TaskDescription({ context }) { + return ( +
+

Context

+ + +

Instruction

+ + +
+ ) +} + +function LocationDescription({ location }) { + if (location == null) { + return ; + } + const loc_str = ("name" in location) ?
+

You are in {location.name}

{location.description}

+
: ; + return ( +
+ {loc_str} +
+ ) +} + +function PersonsaDescription({ personsa }) { + if (personsa == null) { + return ; + } + + function PersonaCards(personsa, personsa_index) { + const caption = (personsa_index === 0) ?

You are {personsa.name}

:

{personsa.name}

; + const special_style = (personsa_index === 0) ? "player" : "npc"; + const full_class = "persona " + special_style; + const desc =

{personsa.persona}

+ return ( +
+ {caption} + {desc} +
+ + ); + } + const elements = personsa.map(PersonaCards); + return (
+ {elements} +
); +} + +function Generating() { + return ( +
+ "Generating ..." +
+ ) +} + +function GeneralDescription() { + return ( +
+

+ In this task you will have a conversation with two other characters in a fantasy game setting. + The other two characters are controlled by Artificial Intelligence (AI). + You will all be given characters and a description of the setting of the conversation. +

+

Chat

+

+ You should play your character, conversing as if you were your character in the provided setting. + The program decides whose turn is next. + When it's your turn, the message bar at the bottom of the right panel is activated and you can play your role. + Otherwise wait for others to play and evaluate their responses. +

+

Evaluate

+

+ After each message from the AI, you will be asked to evaluate the response for its attributes: +

    +
  • Consistent: Does the response 1) make sense in the context of the conversation; 2) make sense in and of itself?
    • +
  • Engaging: Are you engaged by the response? Do you want to continue the conversation?
    • +
  • Out of turn: It didn't make sense for that character to speak at that point?
    • +
  • Mistaken identity: Does it speak like it is someone else?
    • +
  • Contradictory: Contradicts the character's description of what it said before?
    • +
  • Nonsensical: Doesn't make any sense in this context.
    • +
+ You must check at least one box per response, which can be “None” if no attributes apply to the response. +

+
+ ) +} + +function ExtraInformation() { + return ( +
+

What do I talk about?

+

+ Anything, so long as you remain in character. + If it would make sense for your character, you could try to learn about your partners, or talk about yourself, + or the setting you have all been assigned to. +

+
+

When does the task end?

+

+ The conversation will continue for a total of 15 messages. + After reaching that limit you will see the button that allows you to send the chat and submit the HIT. +

+ +
+

What NOT to do:

+
    +
  • Be aware the conversations you have will be made public, so act as you would e.g. on a public social network like Twitter.
    • +
  • Do not talk about the task itself, or about MTurk
    • +
  • Avoid racism, sexism, hate speech, and other forms of inappropriate conversation.
    • +
  • Avoid real world topics and locations, and instead remain in the medieval fantasy setting.
    • +
  • + Don't direct into conversations where you pretend to take actions + (like "here you go! *gives item*"), stick to strictly chat. +
    • +
  • + Don't idle for too long (4 minutes) or you will be disconnected from the chat + and unable to submit. +
    • +
+
+ ) +} \ No newline at end of file diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/styles.css b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/styles.css new file mode 100644 index 00000000000..ef62e4b3fe6 --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/components/styles.css @@ -0,0 +1,24 @@ +.location { + background-color: rgb(232, 147, 147); + padding: 10px; + border-radius: 5px; +} + +.persona { + padding: 10px; + margin-top: 10px; + border-radius: 5px; +} + +.player { + background-color: azure; +} + +.npc { + background-color: bisque; +} + +.rating-response { + background-color: rgb(216 81 30); + width: 20%; +} diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/frontend/main.js b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/main.js new file mode 100644 index 00000000000..aa348f05ce8 --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/frontend/main.js @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * All rights reserved. + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +import React from "react"; +import ReactDOM from "react-dom"; +import "bootstrap-chat/styles.css"; + +import { CustomOnboardingChatApp } from "./components/chat_app_with_onboarding.jsx" +import { TaskDescription } from "./components/sidepane.jsx"; +import { ResponseComponent } from "./components/response_panes.jsx"; +import { RenderChatMessage } from "./components/message.jsx"; + +function MainApp() { + const [needRating, setNeedRating] = React.useState(false); + + function newMessageHandler(messages) { + const lastMessage = messages.at(messages.length - 1); + setNeedRating((lastMessage?.needs_rating === true) ? true : false); + } + + function TextResponse({taskConfig, appSettings, onMessageSend, active}) { + return ( + + ) + } + + return ( + ( + + )} + renderSidePane={({ mephistoContext: { taskConfig }, appContext: { taskContext } }) => ( + + )} + renderTextResponse={ + ({ + mephistoContext: { taskConfig }, + appContext: { appSettings }, + onMessageSend, + active, + + }) => ( + + ) + } + onMessagesChange={(messages) => ( + newMessageHandler(messages))} + /> + ); +} + +ReactDOM.render(, document.getElementById("app")); diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/hydra_configs/conf/multiparty.yaml b/parlai/crowdsourcing/tasks/multi_model_chat/hydra_configs/conf/multiparty.yaml new file mode 100644 index 00000000000..2d043666532 --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/hydra_configs/conf/multiparty.yaml @@ -0,0 +1,39 @@ +#@package _global_ +monitoring_log_rate: 300 +defaults: + - /mephisto/blueprint: model_chat_blueprint + - /mephisto/architect: local + - /mephisto/provider: mock +mephisto: + blueprint: + task_description_file: ${task_dir}/task_config/task_description.html + onboard_task_data_path: ${task_dir}/task_config/onboard_task_data.json + world_file: ${task_dir}/multiparty_worlds.py + annotations_config_path: ${task_dir}/task_config/annotations_config.json + onboarding_qualification: model_chat_onboarding + block_qualification: model_chat_block + allowed_worker_qualification: multiparty-modelchat-allow + chat_data_folder: ${task_dir}/model_chat/ + model_opt_path: ${task_dir}/task_config/model_opts.yaml + custom_source_dir: ${task_dir}/frontend/ + num_conversations: 2 + num_turns: 2 + task_model_parallel: true + check_acceptability: false + include_persona: true + conversation_start_mode: parlai.crowdsourcing.tasks.multi_model_chat.multiparty_worlds + annotation_question: Does this comment from your partner have any of the following attributes? (Check all that apply) + conversations_needed_string: "random_light_prod:2" + override_opt: + context_generator: parlai.crowdsourcing.tasks.multi_model_chat.multiparty_worlds + task: + allowed_concurrent: 2 + assignment_duration_in_seconds: 600 + max_num_concurrent_units: 0 # 0 means infinite; set this to a positive integer to limit concurrent HITs and prevent crashes + maximum_units_per_worker: 3 + task_name: model_chat + task_reward: 3 + task_tags: "chat,conversation,dialog,partner" + task_title: "Chat with a fellow conversationalist!" +mturk: + worker_blocklist_paths: null diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/multiparty_worlds.py b/parlai/crowdsourcing/tasks/multi_model_chat/multiparty_worlds.py new file mode 100644 index 00000000000..dc9b73c4fec --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/multiparty_worlds.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 + +# Copyright (c) Facebook, Inc. and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import json +import os +import numpy as np +import random +import time + +from parlai.core.message import Message +from parlai.core.worlds import validate +import parlai.utils.logging as logging +from parlai.crowdsourcing.tasks.model_chat.utils import Compatibility +from parlai.crowdsourcing.tasks.model_chat.worlds import ( + get_bot_worker, + ModelChatWorld, + ModelChatOnboardWorld, +) + +PERSONA_SETTER_AGENT = 'persona-agent' + + +class MultipartyChatWorld(ModelChatWorld): + def __init__(self, opt, agent, bot, context_info=None): + super(ModelChatWorld, self).__init__(opt, agent=agent, bot=bot) + self.context_info = context_info + self.personas = self.context_info['personas'] + + def _run_initial_turn(self) -> None: + """ + Run the initial turn for both the human and the bot. + + Optionally show the bot its persona. If we are in BST conversation mode, show 2 + previous BST utterances to both the human and the bot; if we are in Meena-like + conversation mode, show "Hi!" to the human and the bot and let the bot respond + accordingly. + """ + # Removing the agents ids for use the id names for the agents. + self.bot.agent_id = None + self.agent.agent_id = None + + if self.opt['include_persona']: + # Sending persona to the bot agent + message = {"episode_done": False} + assert isinstance( + self.personas[1], dict + ), 'Unknown persona format. Check the ContextGenerator in your task.' + message['id'] = PERSONA_SETTER_AGENT + message['text'] = 'PERSONA SETTING MESSAGE' + message['personas'] = self.personas + if 'location' in self.context_info: + message['location'] = self.context_info['location'] + # The bot seeing its persona does not count as a "turn" + self.bot.observe(validate(message), increment_turn=False) + + # Sending persona to the agent + self.agent.observe( + validate( + { + 'episode_done': False, + 'id': PERSONA_SETTER_AGENT, + 'task_data': { + 'personas': self.personas, + 'location': self.context_info['location'], + }, + } + ) + ) + + def has_final_rating(self, act): + return act.get('task_data', {}).get('final_rating') is not None + + def parley(self): + act = None # Adding this for linter errors. + logging.verbose( + f'{self.__class__.__name__}:{self.tag}: is at turn {self.task_turn_idx}, with {self.num_turns} pairs of turns needed...' + ) + + if self.task_turn_idx == 0: + self._run_initial_turn() + self.task_turn_idx += 1 + return + + """Otherwise, we proceed accordingly""" + logging.verbose( + f'{self.__class__.__name__}:{self.tag}: About to act with task turn idx: {self.task_turn_idx}' + ) + + if not self.chat_done: + act = self.bot.act() + human_turn = act.get('human_turn', False) + # Bot decided it is human turn. + + if human_turn: + act = self.agent.act(timeout=self.max_resp_time) + self.chat_done = self.has_final_rating(act) + Compatibility.backward_compatible_force_set( + act, 'id', self.personas[0]['name'] + ) + + act = Message(Compatibility.maybe_fix_act(act)).json_safe_payload() + utterance_data = { + 'agent_idx': 0 if human_turn else 1, + # Get rid of annotations HTML if it's the bot response + 'text': act['text'].split('
')[0], + 'id': act.get('id', 'NULL_ID'), + } + self.dialog.append(utterance_data) + + if human_turn: + self.bot.observe(validate(act)) + else: + act['needs_rating'] = True + self.agent.observe(validate(act)) + + # The new act replaces the old one + act = self.agent.act(timeout=self.max_resp_time) + act.force_set('text', 'THIS IS A RATING ACTION') + p = act['task_data'].get('problem_data_for_prior_message') + if p is not None: + turn_idx = -1 + # Attach the problem data to the last utterance (just generated by bot). + self.__add_problem_data_to_utterance(p, turn_idx=turn_idx) + + self.chat_done = self.has_final_rating(act) + + self.task_turn_idx += 1 + + if self.chat_done: + self.dialog[-1]['final_rating'] = act['task_data']['final_rating'] + + # Save the final chat data + date_folder = time.strftime('%Y_%m_%d') + time_string = time.strftime('%Y%m%d_%H%M%S') + chat_data_subfolder = os.path.join( + self.opt['chat_data_folder'], date_folder + ) + os.makedirs(chat_data_subfolder, exist_ok=True) + chat_data_path = os.path.join( + chat_data_subfolder, + f'{time_string}_{np.random.randint(0, 1000)}_{self.task_type}.json', + ) + self.final_chat_data = self.get_final_chat_data() + self.agent.mephisto_agent.state.messages.append( + { + 'final_chat_data': self.final_chat_data, + 'data': {}, + 'packet_type': None, + 'timestamp': None, + } + ) + # Append the chat data directly to the agent state's message list in + # order to prevent the worker from seeing a new text response in the UI. + # Add some dummy keys for compatibility with all agent state messages + # TODO: remove this when no longer saving data to disk manually + with open(chat_data_path, 'w+') as f_json: + data_str = json.dumps(self.final_chat_data) + f_json.write(data_str) + logging.info( + f'{self.__class__.__name__}:{self.tag}: Data saved at ' + f'{chat_data_path} for model: {self.bot.worker_id}.' + ) + + # Soft-block the worker if there were acceptability violations + acceptability_violations = self.final_chat_data['acceptability_violations'][ + 0 + ] + if acceptability_violations is not None and acceptability_violations != '': + logging.warning( + f'**NOTE** Acceptability violations detected: {acceptability_violations}' + ) + # Grant the failed qualification + self.agent.mephisto_agent.get_worker().grant_qualification( + self.block_qualification, 1 + ) + + def __add_problem_data_to_utterance(self, p, turn_idx: int): + """ + Attach problem data to the bot's prior utterance, given by turn_idx. + + This is copied exactly from the main model_chat world. + """ + logging.verbose(f'Problem matrix:\n{p}') + assert ( + self.dialog[turn_idx]['agent_idx'] == 1 + ), 'Problem data must be attached to a bot utterance.' + assert ( + 'problem_data' not in self.dialog[turn_idx] + ), "Don't overwrite existing problem data!" + self.dialog[turn_idx]['problem_data'] = p + + +class MultiLightModelChatOnboardWorld(ModelChatOnboardWorld): + pass + + +def make_onboarding_world(opt, agent): + return MultiLightModelChatOnboardWorld(opt, agent) + + +def make_world(opt, agents): + + # Extract important components from opt + statistics_condition = opt['statistics_condition'] + context_generator = opt['context_generator'] + + # Get context: personas, previous utterances, etc. + if context_generator is not None: + context_info = context_generator.get_context() + else: + context_info = None + + # Decide on a bot to use + run_statistics = opt['run_statistics'] + with statistics_condition: + remaining_counts_needed = [ + (m, c - run_statistics[m]) for (m, c) in opt['conversations_needed'].items() + ] + remaining_counts_needed.sort(reverse=True, key=lambda x: x[1]) + model_name = remaining_counts_needed[0][0] + print(f'Remaining conversation counts needed: {remaining_counts_needed}') + print(f'Choosing the "{model_name}" model for the bot.') + bot_worker = get_bot_worker(opt=opt, model_name=model_name) + + return MultipartyChatWorld( + opt, agent=agents[0], bot=bot_worker, context_info=context_info + ) + + +def get_world_params(): + return {"agent_count": 1} + + +def get_settings(opt=None): + """ + Returns the conversation settings. + + This is a place-holder function with a few hand selected settings, override with + more for real data collection. + """ + return [ + { + 'personas': [ + { + 'name': 'grass snake', + 'persona': "I'm a grass snake. I slither around the castle and fields. I eat the rodents that eat the grain.", + }, + { + 'name': 'tribesman', + 'persona': ( + "I am a tribesman in my group. I am known as a leader in my community and love to help my people. " + " I'm very level headed and don't get angry easily. Many of my peers come to me to solve disagreements." + ), + }, + { + 'name': 'thief', + 'persona': ( + 'I live alone in a tent in the woods. I steal food from the townspeople and coal from the blacksmith.' + ' The village police can not find me to put me in jail.' + ), + }, + ], + 'location': { + 'name': 'Bamboo hut', + 'description': ( + "Built of bamboo trunks and a bamboo leaf roof, this small hut has one window on each side and a short door," + " where those who enter must stoop down so they don't hit their heads. " + "A dirt floor is covered with palm fronds gathered from the jungle; " + "four small rocks are placed around the center of the room, forming a place for the occupants to sit. " + "A small fire burns just outside of the hut, and a wooden spit is suspended over the fire. " + "One of the support poles of the hut has a woven grass bag hanging from it. The bag contains a half-dozen coconuts," + " clearly gathered for consuming at a later time. A colorful lizard is sleeping in the sun in one of the windows." + ), + }, + }, + { + 'personas': [ + { + 'name': 'clergy', + 'persona': ( + "I oversee the castle's chapel. I collect alms for the poor. " + "I am the spiritual leader of the subjects of the kingdom." + ), + }, + { + 'name': 'Nuns', + 'persona': ( + "I am a nun and I live in a monastery with others nuns and fathers who server the king." + " I pray to the lord everyday that Queen remains in good health. " + "I was made a sister at a young age and didn't have a choice. " + "I will never know what being with a man will feel like." + ), + }, + { + 'name': 'priest', + 'persona': 'I am here to help the needy. I am well respected in the town. I can not accept lying.', + }, + ], + 'location': { + 'name': 'Church Entryway', + 'description': ( + 'The church has marble floors and a huge frost window. ' + 'There are benches made from wood and a big organ can be seen at the front stage.' + ' There is gold trim all around the church.' + ), + }, + }, + ] + + +class ContextGenerator: + """ + Generates contexts shown to crowdsourced workers during data collection. + """ + + def __init__(self, opt, datatype: str = 'test', seed: int = None): + """ + Initalize the context generator. + """ + if seed is not None: + self.rng = random.Random(seed) + else: + self.rng = random.Random() + + def get_context(self) -> dict: + """ + Get context information to be shown at the beginning of one conversation. Values + in return dict: + + - context_dataset: the dataset + - personas: a list of dict where each dictionary is a persona as stored in this task messages. + """ + setting = random.choice(get_settings()) + return { + 'context_dataset': 'multi-modelchat', + 'personas': setting['personas'], + 'location': setting['location'], + } diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/task_config/annotations_config.json b/parlai/crowdsourcing/tasks/multi_model_chat/task_config/annotations_config.json new file mode 100644 index 00000000000..5f660a6cfa3 --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/task_config/annotations_config.json @@ -0,0 +1,33 @@ +{ + "config": { + "consistent": { + "name": "Consistent", + "description": "Does the response 1) make sense in the context of the conversation; 2) make sense in and of itself?" + }, + "engaging": { + "name": "Engaging", + "description": "Are you engaged by the response? Do you want to continue the conversation?" + }, + "out_of_turn": { + "name": "Out of turn", + "description": "It didn't make sense for that character to speak at that point?" + }, + "identity": { + "name": "Mistaken identity", + "description": "Does it speak like it is someone else?" + }, + "contradictory": { + "name": "Contradictory", + "description": "Contradicts the character's desciption of what it said before?" + }, + "nonsensical": { + "name": "Nonsensical", + "description": "Doesn't make any sense in this context." + }, + "none": { + "name": "None", + "description": "A generic response that doesn't fit any of the other options." + } + }, + "show_line_breaks": false +} diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/task_config/model_opts.yaml b/parlai/crowdsourcing/tasks/multi_model_chat/task_config/model_opts.yaml new file mode 100644 index 00000000000..f75e61a98d2 --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/task_config/model_opts.yaml @@ -0,0 +1,5 @@ +random_light_prod: > + --model parlai.crowdsourcing.tasks.multi_model_chat.agents:MultipartyModelChatAgent + --decision-agent parlai.crowdsourcing.tasks.multi_model_chat.agents:RandomSpeakerDecicionsAgent + --speech-model-file zoo:light_whoami/profile_expanded_attention_128/model + --context-format light diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/task_config/onboard_task_data.json b/parlai/crowdsourcing/tasks/multi_model_chat/task_config/onboard_task_data.json new file mode 100644 index 00000000000..bb490d899bb --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/task_config/onboard_task_data.json @@ -0,0 +1,67 @@ +{ + "dialog": [ + [ + { + "text": "Nice to see you stranger. I am a traveler coming from far lands.", + "agent_idx": 0, + "id": "human_evaluator", + "episode_done": false + }, + { + "text": "Welcome to our village. I am the innkeeper here, do you need rest?", + "agent_idx": 1, + "episode_done": false, + "id": "model", + "answers": ["consistent", "engaging"] + } + ], + [ + { + "text": "My gratitudes. Indeed I am in dire need of a room to rest.", + "agent_idx": 0, + "id": "human_evaluator", + "episode_done": false + }, + { + "text": "I am also a merchant coming to this village for the first time. Do you know any place for us to rest?", + "agent_idx": 1, + "episode_done": false, + "id": "model", + "answers": ["identity", "contradictory"] + } + ], + [ + { + "text": "", + "agent_idx": 0, + "id": "human_evaluator", + "episode_done": false + }, + { + "text": "Oh, I am sorry. What I said was a mistake, I am sometimes very confused. So tell me, how long do you intend to stay in our village.", + "agent_idx": 1, + "episode_done": false, + "id": "model", + "answers": ["consistent", "engaging"] + } + ], + [ + { + "text": "", + "agent_idx": 0, + "id": "human_evaluator", + "episode_done": false + }, + { + "text": "I am happy to hear that.", + "agent_idx": 1, + "episode_done": false, + "id": "model", + "answers": ["out_of_turn"] + } + ] + ], + "min_correct": 2, + "max_incorrect": 2, + "max_failures_allowed": 3 +} \ No newline at end of file diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/task_config/task_description.html b/parlai/crowdsourcing/tasks/multi_model_chat/task_config/task_description.html new file mode 100644 index 00000000000..6f457fca28f --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/task_config/task_description.html @@ -0,0 +1,50 @@ +

What is this task?

+ +

+ In this task you will have a conversation with two other characters in a fantasy game setting. + The other two characters are controlled by Artificial Intelligence (AI). + You will all be given characters and a description of the setting of the conversation. + +

+

Chat

+

+ You should play your character, conversing as if you were your character in the provided setting. + The program decides whose turn is next. + When it's your turn, the message bar at the bottom of the right panel is activated and you can play your role. + Otherwise wait for others to play and evaluate their responses. +

+

Evaluate

+

+ After each message from the AI, you will be asked to evaluate the response for attributes such as: + Consistent, Engaging, Out of turn, Mistaken identity, Contradictory, Nonsensical. + You must check at least one box per response, which can be “None” if no attributes apply to the response. +

+
+

What do I talk about?

+

+ Anything, so long as you remain in character. + If it would make sense for your character, you could try to learn about your partners, or talk about yourself, + or the setting you have all been assigned to. +

+
+

When does the task end?

+

+ The conversation ends when all characters have spoken at least 15 turns; + counting all the messages from you and the AI characters. +

+ +
+

What NOT to do:

+
    +
  • Do not talk about the task itself, or about MTurk
    • +
  • Avoid racism, sexism, hate speech, and other forms of inappropriate conversation.
    • +
  • Avoid real world topics and locations, and instead remain in the medieval fantasy setting.
    • +
  • + Don't direct into conversations where you pretend to take actions + (like "here you go! *gives item*"), stick to strictly chat. +
    • +
  • + Don't idle for too long (5 minutes) or you will be disconnected from the chat + and unable to submit. +
    • +
\ No newline at end of file