Skip to content
This repository has been archived by the owner on Nov 3, 2023. It is now read-only.

Commit

Permalink
Adding option for file with annotation indices per conversation and o…
Browse files Browse the repository at this point in the history
…ther features to turn annotations static (#3154)

* black changes

* black

* fixed linting
  • Loading branch information
mwillwork authored Oct 12, 2020
1 parent 8506932 commit d736133
Show file tree
Hide file tree
Showing 7 changed files with 216 additions and 41 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ class LaunchConfig:
command line as params)
"""

BLUEPRINT_TYPE = 'turn_annotations_static_inflight_qa_blueprint'

# Run the below to register the requester if needed (first time only)
# mephisto register mturk_sandbox --name:XXXX
# --access-key-id:XXXX --secret-access-key:XXXX
Expand Down Expand Up @@ -47,5 +49,5 @@ class LaunchConfig:
WORKER_BLOCK_LIST = []

# Blueprint specific params
ANNOTATE_LAST_UTTERANCE_ONLY = False
ASK_REASON = False
ANNOTATION_INDICES_JSONL = None
34 changes: 30 additions & 4 deletions parlai/crowdsourcing/tasks/turn_annotations_static/run_task.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import shutil
import subprocess
import random
import logging

from parlai.core.params import ParlaiParser
from parlai.core.script import ParlaiScript
Expand Down Expand Up @@ -59,16 +60,16 @@ def setup_mephisto(launch_config):
architect_type, requester_name, db, args = parser.parse_launch_arguments()

arg_string = (
"--blueprint-type turn_annotations_static_inflight_qa_blueprint "
f"--blueprint-type {launch_config.BLUEPRINT_TYPE} "
f"--architect-type {architect_type} "
f"--requester-name {requester_name} "
f'--task-title "\\"{launch_config.TASK_TITLE}\\"" '
f'--task-description "\\"{launch_config.TASK_DESCRIPTION}\\"" '
f"--task-name turn-ann-s "
f'--task-name {launch_config.TASK_NAME} '
f'--task-source "{TASK_DIRECTORY}/webapp/build/bundle.js" '
f'--task-reward {launch_config.TASK_REWARD} '
f'--subtasks-per-unit {launch_config.SUBTASKS_PER_UNIT} '
f'--annotate-last-utterance-only {launch_config.ANNOTATE_LAST_UTTERANCE_ONLY} '
f'--annotation-buckets {launch_config.ANNOTATION_BUCKETS} '
f'--ask-reason {launch_config.ASK_REASON} '
f'--task-tags chat,conversation,dialog,partner '
# How many workers to do each assignment
Expand All @@ -82,6 +83,32 @@ def setup_mephisto(launch_config):
f'--onboarding-qualification turn-ann-s-onb '
f"-use-onboarding True "
)
# Optional flags:
try:
arg_string += (
f'--annotation-question "\\"{launch_config.ANNOTATION_QUESTION}\\"" '
)
except Exception:
logging.info(f'Launch config {launch_config} had no ANNOTATION_QUESTION field')

try:
arg_string += (
f'--annotation-indices-jsonl {launch_config.ANNOTATION_INDICES_JSONL} '
)
except Exception:
logging.info(f'Launch config {launch_config} had no ANNOTATION_INDICES_JSONL')

try:
arg_string += f'--conversation-count {launch_config.CONVERSATION_COUNT} '
except Exception:
logging.info(f'Launch config {launch_config} had no CONVERSATION_COUNT')

try:
arg_string += f'--onboarding-data {launch_config.ONBOARDING_DATA} '
except Exception:
logging.info(f'Launch config {launch_config} had no ONBOARDING_DATA')

print(arg_string)
return db, arg_string


Expand Down Expand Up @@ -145,7 +172,6 @@ def run_task(opt):

build_task()
operator = Operator(db)
print(f'ARG_STRING: {arg_string}')
operator.parse_and_launch_run_wrapper(shlex.split(arg_string), extra_args={})
operator.wait_for_runs_then_shutdown()

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{"data": [1, 3, 5]}
{"data": [1]}
{"data": [7]}
{"data": [9, 11]}
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
# LICENSE file in the root directory of this source tree.

import os
import random
import numpy as np
import math
import json
import logging
Expand Down Expand Up @@ -38,22 +40,55 @@ class TurnAnnotationsStaticBlueprint(StaticReactBlueprint):

def __init__(self, task_run: "TaskRun", opts: Any):
super().__init__(task_run, opts)
self.subtasks_per_unit = opts['subtasks_per_unit']
print(f'Running {self.__class__.__name__} with opts: {self.opts}')
random.seed(self.opts["random_seed"])
np.random.seed(self.opts["random_seed"])
self.subtasks_per_unit = self.opts['subtasks_per_unit']
self.conversation_count = self.opts['conversation_count']

if self.subtasks_per_unit <= 0:
raise Exception(
f'subtasks-per-unit must be greater than zero but was {self.subtasks_per_unit}'
)
grouped_data = []

self.raw_data = self._initialization_data_dicts

# Load from file if needed specifying which utterances within each
# conversation to annotate
self.annotation_indices = None
if self.opts['annotation_indices_jsonl']:
self.annotation_indices = []
with open(
self.opts['annotation_indices_jsonl'], "r", encoding="utf-8-sig"
) as f:
line = f.readline()
while line:
conversation_indices = json.loads(line)
self.annotation_indices.append(conversation_indices)
line = f.readline()
if len(self.annotation_indices) != len(self.raw_data):
raise Exception(
f'Cannot specify a different length of annotation indices ({len(self.annotation_indices)}) than conversations ({len(self.raw_data)}).'
)
# TODO: should check that utterances specified are all bot
# utterances (agent_idx == 1)

if self.conversation_count:
self.raw_data = self.raw_data[: self.conversation_count]
if self.annotation_indices:
self.annotation_indices = self.annotation_indices[
: self.conversation_count
]

# Reorganize the self-chat data
self._initialization_data_dicts = self.process_data(
self._initialization_data_dicts
self.raw_data, annotation_indices=self.annotation_indices
)

# Now chunk the data into groups of <num_subtasks>
grouped_data = []
logging.info(
f'Raw data length: {len(self._initialization_data_dicts)}. self.subtasks_per_unit: {self.subtasks_per_unit}'
f'Raw data length: {len(self.raw_data)}. self.subtasks_per_unit: {self.subtasks_per_unit}'
)
for i in range(0, len(self._initialization_data_dicts), self.subtasks_per_unit):
chunk = self._initialization_data_dicts[i : i + self.subtasks_per_unit]
Expand All @@ -70,6 +105,20 @@ def add_args_to_group(cls, group: "ArgumentGroup") -> None:
Adds required options for TurnAnnotationStaticBlueprint.
"""
super().add_args_to_group(group)
group.add_argument(
"--random-seed",
dest="random_seed",
type=int,
default=42,
help="seed for random",
)
group.add_argument(
"--annotation-question",
dest="annotation_question",
type=str,
default='Does this comment require any annotations? (Check all that apply)',
help="The string displayed above the checkboxes for each annotation in the task.",
)
group.add_argument(
"--subtasks-per-unit",
dest="subtasks_per_unit",
Expand All @@ -78,11 +127,11 @@ def add_args_to_group(cls, group: "ArgumentGroup") -> None:
help="number of subtasks/comparisons to do per unit",
)
group.add_argument(
"--annotate-last-utterance-only",
dest="annotate_last_utterance_only",
type=str2bool, # Need to handle it being 'False' in arg_string
default=False,
help="If we only want the crowdworker to annotate the last utterance in the conversation",
"--annotation-indices-jsonl",
dest="annotation_indices_jsonl",
type=str,
default=None,
help="Specify which utterance indices to annotate per conversation in a JSONL file. Must be same length as conversations data-jsonl file. See example file in task_config/annotation_indices_example.jsonl",
)
group.add_argument(
"--ask-reason",
Expand All @@ -91,6 +140,13 @@ def add_args_to_group(cls, group: "ArgumentGroup") -> None:
default=False,
help="If we want to ask the crowdworker for a reason for each of their annotations in a text field",
)
group.add_argument(
"--conversation-count",
dest="conversation_count",
type=int,
default=None,
help="Specify a positive integer if you want to use only the first N conversations in the data file",
)
group.add_argument(
"--onboarding-data",
dest="onboarding_data",
Expand Down Expand Up @@ -123,33 +179,101 @@ def get_frontend_args(self) -> Dict[str, Any]:
return {
"task_description": self.opts['task_description'],
"task_title": self.opts['task_title'],
"annotation_question": self.opts['annotation_question'],
"onboarding_data": onboarding_data,
"annotation_buckets": annotation_buckets,
"annotate_last_utterance_only": self.opts['annotate_last_utterance_only'],
"ask_reason": self.opts['ask_reason'],
"frame_height": '100%',
"num_subtasks": self.opts["subtasks_per_unit"],
"block_mobile": True,
}

def process_data(self, data_dicts):
def process_data(self, data_dicts, annotation_indices=None):
"""
Override this in a subclass if you want to change how data is processed from
input file before being sent to the frontend.
"""
output = []
for d in data_dicts:
new_dialogue = []
for utt in d['dialog']:
# If there is a persona, which is context as in the ConvAI2
# task, we don't want to display the persona utterances
if 'persona' not in utt[0]['text']:
new_dialogue.append({'text': utt[0]['text'], 'agent_idx': 0})
if 'persona' not in utt[1]['text']:
new_dialogue.append({'text': utt[1]['text'], 'agent_idx': 1})
output.append(new_dialogue)
total_annotation_count = 0
for conv_idx, d in enumerate(data_dicts):
max_turn_to_show = len(d['dialog']) - 1
if annotation_indices:
total_annotation_count += len(annotation_indices[conv_idx])
# We only want to show the conversation up to the last
# utterance we need annotations on, b/c otherwise may confuse
# or bias the turkers
if len(annotation_indices[conv_idx]) > 1:
logging.info(
f'Splitting {len(annotation_indices[conv_idx])} separate problematic utterance annotations in the same conversation into two separate conversations for this task. This avoids biasing the turkers with utterances that may come after one of the annotations.'
)
for a in annotation_indices[conv_idx]:
processed_dialog = self._process_conversation(d, [a])
output.append(processed_dialog)
else:
processed_dialog = self._process_conversation(d, [max_turn_to_show])
output.append(processed_dialog)
print(
f'Processed {len(data_dicts)} total conversations into {len(output)} conversations to be used in crowdsourcing task with {total_annotation_count} total annotations.'
)
np.random.shuffle(output)
return output

def _process_conversation(self, d, annotation_indices):
"""
Helper function for processing conversations.
:param annotation_indices:
Array of turn indices to annotate of the
actual conversation not including the context [So 0 is the "Hi!" if
that's the first non-context utterance of the conversation.]
:return: modified dialogue object
"""
new_dialogue = []
max_turn_to_show = max(annotation_indices)
adjusted_turn_idx = 0
for full_turn in d['dialog']:
if len(full_turn) != 2:
print(
f'Warning! Skipping incomplete conversation! full_turn was: {full_turn}'
)
continue
if adjusted_turn_idx > max_turn_to_show:
logging.info(
f'Skipping {adjusted_turn_idx}th utterance, b/c max_turn_to_show was {max_turn_to_show}.'
)
continue
# If there is a persona, which is context as in the ConvAI2
# task, we don't want to display the persona utterances
if 'persona' not in full_turn[0]['text']:
do_annotate = False
new_dialogue.append(
{
'text': full_turn[0]['text'],
'agent_idx': 0,
'do_annotate': do_annotate,
'other_metadata': full_turn[0]['other_metadata'],
}
)
adjusted_turn_idx += 1
if 'persona' not in full_turn[1]['text']:
do_annotate = True
if annotation_indices:
do_annotate = adjusted_turn_idx in annotation_indices
new_dialogue.append(
{
'text': full_turn[1]['text'],
'agent_idx': 1,
'do_annotate': do_annotate,
'other_metadata': full_turn[1]['other_metadata'],
}
)
adjusted_turn_idx += 1
if adjusted_turn_idx < max_turn_to_show:
raise Exception(
f'Conversation had {adjusted_turn_idx} but max_turn_to_show was {max_turn_to_show}'
)
return new_dialogue


@register_mephisto_abstraction()
class TurnAnnotationsStaticInFlightQABlueprint(TurnAnnotationsStaticBlueprint):
Expand All @@ -171,10 +295,17 @@ def __init__(self, task_run: "TaskRun", opts: Any):
qc_convo = json.loads(line)
raw_qc_convos.append(qc_convo)
line = f.readline()
self.quality_control_convos = self.process_data(raw_qc_convos)
# Annotate all the utterances in the quality controls
# So annotation_indices=None here
self.quality_control_convos = self.process_data(
raw_qc_convos, annotation_indices=None
)

# Re-chunk the data to add a quality control convo as the last subtask
# No shuffling of the data for reproducibility's sake
# (quality control will always be last subtask)
# TODO: I don't think we need to re-chunk this actually; just iterate
# over the data and add the quality control task
all_data = []
for grp in self._initialization_data_dicts:
all_data.extend(grp)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@ function TaskFrontend({ taskData, taskConfig, isOnboarding, onSubmit }) {
return <LoadingScreen />;
}
if (isOnboarding) {
return <OnboardingComponent onboardingData={taskConfig.onboarding_data} annotationBuckets={taskConfig.annotation_buckets} onSubmit={onSubmit} />;
return <OnboardingComponent onboardingData={taskConfig.onboarding_data} annotationBuckets={taskConfig.annotation_buckets} annotationQuestion={taskConfig.annotation_question} onSubmit={onSubmit} />;
}
return (
<MainTaskComponent taskData={taskData} annotationBuckets={taskConfig.annotation_buckets} taskTitle={taskConfig.task_title} taskDescription={taskConfig.task_description} taskConfig={taskConfig} onSubmit={onSubmit}></MainTaskComponent>
<MainTaskComponent taskData={taskData} taskTitle={taskConfig.task_title} taskDescription={taskConfig.task_description} taskConfig={taskConfig} onSubmit={onSubmit}></MainTaskComponent>
);
}

Expand Down
Loading

0 comments on commit d736133

Please sign in to comment.