From 33e9941df062ac02a75d0d85d3615e64d892b6c4 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Thu, 23 Dec 2021 11:44:48 -0500 Subject: [PATCH 1/4] Layer changes --- .../crowdsourcing/tasks/model_chat/README.md | 2 +- .../frontend/components/checkboxes.jsx | 56 ++++++++++--------- .../task_config/annotations_config.json | 3 +- 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/parlai/crowdsourcing/tasks/model_chat/README.md b/parlai/crowdsourcing/tasks/model_chat/README.md index 875c4c5341c..73364362c82 100644 --- a/parlai/crowdsourcing/tasks/model_chat/README.md +++ b/parlai/crowdsourcing/tasks/model_chat/README.md @@ -15,7 +15,7 @@ Set `mephisto.blueprint.chat_data_folder` to the root folder that you want all r ## Passing in task config files The following flags can be passed in to specify filepaths for overriding the text shown to the workers and the settings of the annotation categories. If they are not specified, the defaults in the `task_config/` folder will be used. -- `mephisto.blueprint.annotations_config_path`: JSON file configuring annotation categories. Set this flag to `""` to disable annotation of model responses. +- `mephisto.blueprint.annotations_config_path`: JSON file configuring annotation categories. Set this flag to `""` to disable annotation of model responses. TODO: say what the `"show_line_breaks"` field does! - `mephisto.blueprint.left_pane_text_path`: HTML to show on the left-hand pane of the chat window. - `mephisto.blueprint.onboard_task_data_path`: JSON specifying parameters for testing workers during onboarding. Onboarding is only run if model responses will be annotated. - `mephisto.blueprint.task_description_file`: HTML to show on the initial task-description page shown to the worker. diff --git a/parlai/crowdsourcing/tasks/model_chat/frontend/components/checkboxes.jsx b/parlai/crowdsourcing/tasks/model_chat/frontend/components/checkboxes.jsx index 79379d77854..3deda1474b3 100644 --- a/parlai/crowdsourcing/tasks/model_chat/frontend/components/checkboxes.jsx +++ b/parlai/crowdsourcing/tasks/model_chat/frontend/components/checkboxes.jsx @@ -8,13 +8,13 @@ import React from "react"; -function Checkboxes({ - annotationBuckets, - turnIdx, - askReason, - annotations, - onUpdateAnnotations, - enabled=true, +function Checkboxes({ + annotationBuckets, + turnIdx, + askReason, + annotations, + onUpdateAnnotations, + enabled=true, }) { var reasonComponent = (
@@ -30,27 +30,32 @@ function Checkboxes({ } // TODO: add support for radio input type let input_type = "checkbox"; + const showLineBreaks = annotationBuckets.hasOwnProperty("show_line_breaks") ? annotationBuckets.show_line_breaks : false; + const numBuckets = Object.keys(annotationBuckets.config).length; return (
{ - Object.keys(annotationBuckets.config).map(c => ( - - { - let newVal = evt.target.checked; - let oldAnnotations = Object.assign({}, annotations); - oldAnnotations[c] = newVal; - onUpdateAnnotations(oldAnnotations); - }} - disabled={!enabled} - /> - - {annotationBuckets.config[c].name} + Object.keys(annotationBuckets.config).map((c, checkboxIdx) => ( + <> + + { + let newVal = evt.target.checked; + let oldAnnotations = Object.assign({}, annotations); + oldAnnotations[c] = newVal; + onUpdateAnnotations(oldAnnotations); + }} + disabled={!enabled} + /> + + {annotationBuckets.config[c].name} + - + {(showLineBreaks && checkboxIdx < numBuckets - 1) ?

: ''} + )) }
@@ -58,5 +63,6 @@ function Checkboxes({
) } +// showLineBreaks: show a line break after every checkbox other than the final one -export { Checkboxes }; \ No newline at end of file +export { Checkboxes }; diff --git a/parlai/crowdsourcing/tasks/model_chat/task_config/annotations_config.json b/parlai/crowdsourcing/tasks/model_chat/task_config/annotations_config.json index 8d028c07a12..06917562c24 100644 --- a/parlai/crowdsourcing/tasks/model_chat/task_config/annotations_config.json +++ b/parlai/crowdsourcing/tasks/model_chat/task_config/annotations_config.json @@ -24,5 +24,6 @@ "name": "None, all good", "description": "This response implies that there are no problems with the data" } - } + }, + "show_line_breaks": false } From e14803eafe64bacebf6b827adbea14afb4e538a1 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Thu, 23 Dec 2021 16:51:24 -0500 Subject: [PATCH 2/4] Docs --- parlai/crowdsourcing/tasks/model_chat/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/parlai/crowdsourcing/tasks/model_chat/README.md b/parlai/crowdsourcing/tasks/model_chat/README.md index 73364362c82..598884d42ea 100644 --- a/parlai/crowdsourcing/tasks/model_chat/README.md +++ b/parlai/crowdsourcing/tasks/model_chat/README.md @@ -15,7 +15,9 @@ Set `mephisto.blueprint.chat_data_folder` to the root folder that you want all r ## Passing in task config files The following flags can be passed in to specify filepaths for overriding the text shown to the workers and the settings of the annotation categories. If they are not specified, the defaults in the `task_config/` folder will be used. -- `mephisto.blueprint.annotations_config_path`: JSON file configuring annotation categories. Set this flag to `""` to disable annotation of model responses. TODO: say what the `"show_line_breaks"` field does! +- `mephisto.blueprint.annotations_config_path`: JSON file configuring annotation categories. + - Set this flag to `""` to disable annotation of model responses. + - If the text of the annotation categories is very long, you can set the `"show_line_breaks"` field (see `task_config/annotations_config.json`) to `true` in order to put a line break between the checkbox and text string of each category. - `mephisto.blueprint.left_pane_text_path`: HTML to show on the left-hand pane of the chat window. - `mephisto.blueprint.onboard_task_data_path`: JSON specifying parameters for testing workers during onboarding. Onboarding is only run if model responses will be annotated. - `mephisto.blueprint.task_description_file`: HTML to show on the initial task-description page shown to the worker. From 13e38866054fbfaa47c186f1da24cb01f53b3b2e Mon Sep 17 00:00:00 2001 From: EricMichaelSmith Date: Thu, 23 Dec 2021 17:12:31 -0500 Subject: [PATCH 3/4] Fixes --- .../tasks/model_chat/expected_states/final_chat_data.json | 2 +- .../crowdsourcing/tasks/model_chat/expected_states/state.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data.json b/tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data.json index a8d43fb8bbd..900700713d2 100644 --- a/tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data.json +++ b/tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data.json @@ -132,7 +132,7 @@ "1" ], "task_description": { - "annotations_config": "{\n \"config\": {\n \"bucket_0\": {\n \"name\": \"Bucket 0\",\n \"description\": \"this response implies something...0\"\n },\n \"bucket_1\": {\n \"name\": \"Bucket 1\",\n \"description\": \"this response implies something...1\"\n },\n \"bucket_2\": {\n \"name\": \"Bucket 2\",\n \"description\": \"this response implies something...2\"\n },\n \"bucket_3\": {\n \"name\": \"Bucket 3\",\n \"description\": \"this response implies something...3\"\n },\n \"bucket_4\": {\n \"name\": \"Bucket 4\",\n \"description\": \"this response implies something...4\"\n },\n \"none_all_good\": {\n \"name\": \"None, all good\",\n \"description\": \"This response implies that there are no problems with the data\"\n }\n }\n}\n", + "annotations_config": "{\n \"config\": {\n \"bucket_0\": {\n \"name\": \"Bucket 0\",\n \"description\": \"this response implies something...0\"\n },\n \"bucket_1\": {\n \"name\": \"Bucket 1\",\n \"description\": \"this response implies something...1\"\n },\n \"bucket_2\": {\n \"name\": \"Bucket 2\",\n \"description\": \"this response implies something...2\"\n },\n \"bucket_3\": {\n \"name\": \"Bucket 3\",\n \"description\": \"this response implies something...3\"\n },\n \"bucket_4\": {\n \"name\": \"Bucket 4\",\n \"description\": \"this response implies something...4\"\n },\n \"none_all_good\": {\n \"name\": \"None, all good\",\n \"description\": \"This response implies that there are no problems with the data\"\n }\n },\n \"show_line_breaks\": false\n}\n", "model_nickname": "fixed_response", "model_file": "/private/home/ems/GitHub/facebookresearch/ParlAI_dev/data/models/fixed_response/model", "model_opt": { diff --git a/tests/crowdsourcing/tasks/model_chat/expected_states/state.json b/tests/crowdsourcing/tasks/model_chat/expected_states/state.json index 8f4d83c082c..1f5fdedee05 100644 --- a/tests/crowdsourcing/tasks/model_chat/expected_states/state.json +++ b/tests/crowdsourcing/tasks/model_chat/expected_states/state.json @@ -357,7 +357,7 @@ "person2_seed_utterance": null, "personas": null, "task_description": { - "annotations_config": "{\n \"config\": {\n \"bucket_0\": {\n \"name\": \"Bucket 0\",\n \"description\": \"this response implies something...0\"\n },\n \"bucket_1\": {\n \"name\": \"Bucket 1\",\n \"description\": \"this response implies something...1\"\n },\n \"bucket_2\": {\n \"name\": \"Bucket 2\",\n \"description\": \"this response implies something...2\"\n },\n \"bucket_3\": {\n \"name\": \"Bucket 3\",\n \"description\": \"this response implies something...3\"\n },\n \"bucket_4\": {\n \"name\": \"Bucket 4\",\n \"description\": \"this response implies something...4\"\n },\n \"none_all_good\": {\n \"name\": \"None, all good\",\n \"description\": \"This response implies that there are no problems with the data\"\n }\n }\n}\n", + "annotations_config": "{\n \"config\": {\n \"bucket_0\": {\n \"name\": \"Bucket 0\",\n \"description\": \"this response implies something...0\"\n },\n \"bucket_1\": {\n \"name\": \"Bucket 1\",\n \"description\": \"this response implies something...1\"\n },\n \"bucket_2\": {\n \"name\": \"Bucket 2\",\n \"description\": \"this response implies something...2\"\n },\n \"bucket_3\": {\n \"name\": \"Bucket 3\",\n \"description\": \"this response implies something...3\"\n },\n \"bucket_4\": {\n \"name\": \"Bucket 4\",\n \"description\": \"this response implies something...4\"\n },\n \"none_all_good\": {\n \"name\": \"None, all good\",\n \"description\": \"This response implies that there are no problems with the data\"\n }\n },\n \"show_line_breaks\": false\n}\n", "model_nickname": "fixed_response", "model_opt": { "init_opt": null, @@ -515,7 +515,7 @@ "hit_ids": ["1"], "assignment_ids": ["1"], "task_description": { - "annotations_config": "{\n \"config\": {\n \"bucket_0\": {\n \"name\": \"Bucket 0\",\n \"description\": \"this response implies something...0\"\n },\n \"bucket_1\": {\n \"name\": \"Bucket 1\",\n \"description\": \"this response implies something...1\"\n },\n \"bucket_2\": {\n \"name\": \"Bucket 2\",\n \"description\": \"this response implies something...2\"\n },\n \"bucket_3\": {\n \"name\": \"Bucket 3\",\n \"description\": \"this response implies something...3\"\n },\n \"bucket_4\": {\n \"name\": \"Bucket 4\",\n \"description\": \"this response implies something...4\"\n },\n \"none_all_good\": {\n \"name\": \"None, all good\",\n \"description\": \"This response implies that there are no problems with the data\"\n }\n }\n}\n", + "annotations_config": "{\n \"config\": {\n \"bucket_0\": {\n \"name\": \"Bucket 0\",\n \"description\": \"this response implies something...0\"\n },\n \"bucket_1\": {\n \"name\": \"Bucket 1\",\n \"description\": \"this response implies something...1\"\n },\n \"bucket_2\": {\n \"name\": \"Bucket 2\",\n \"description\": \"this response implies something...2\"\n },\n \"bucket_3\": {\n \"name\": \"Bucket 3\",\n \"description\": \"this response implies something...3\"\n },\n \"bucket_4\": {\n \"name\": \"Bucket 4\",\n \"description\": \"this response implies something...4\"\n },\n \"none_all_good\": {\n \"name\": \"None, all good\",\n \"description\": \"This response implies that there are no problems with the data\"\n }\n },\n \"show_line_breaks\": false\n}\n", "model_nickname": "fixed_response", "model_file": null, "model_opt": { From 238ee0d10cc4a9ca9a00153b8bc13960308bb612 Mon Sep 17 00:00:00 2001 From: EricMichaelSmith Date: Thu, 23 Dec 2021 17:32:12 -0500 Subject: [PATCH 4/4] Empty