From 9788e5713e31da1c3f49275630085714ac9ebdf3 Mon Sep 17 00:00:00 2001
From: Eric Smith <EricMichaelSmith@users.noreply.github.com>
Date: Thu, 23 Dec 2021 11:49:53 -0500
Subject: [PATCH 1/6] Layer changes

---
 .../frontend/components/response_panes.jsx    | 144 ++++++++++++------
 .../tasks/model_chat/model_chat_blueprint.py  |   7 +-
 2 files changed, 101 insertions(+), 50 deletions(-)

diff --git a/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx b/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx
index 7f96f99c72c..cc71ab44286 100644
--- a/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx
+++ b/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx
@@ -8,13 +8,7 @@
 
 import React from "react";
 
-import { 
-  FormControl, 
-  Button,
-  Col,
-  FormGroup,
-  ControlLabel,
-} from "react-bootstrap";
+import { Button, Col, ControlLabel, Form, FormControl, FormGroup } from "react-bootstrap";
 
 
 function hasAnyAnnotations(annotations) {
@@ -28,26 +22,8 @@ function hasAnyAnnotations(annotations) {
   }
   return false;
 }
-  
-function FinalSurvey({ taskConfig, onMessageSend, active, currentCheckboxes}) {
-  const [rating, setRating] = React.useState(0);
-  const [sending, setSending] = React.useState(false);
-
-  const tryMessageSend = React.useCallback(() => {
-    if (active && !sending) {
-      setSending(true);
-      onMessageSend({ 
-        text: "", 
-        task_data: {
-          problem_data_for_prior_message: currentCheckboxes,
-          final_rating: rating,
-        },
-      }).then(() => {
-        setSending(false);
-      });
-    }
-  }, [active, sending, rating, onMessageSend]);
 
+function RatingSelector({ active, ratings, sending, ratingQuestion, ratingIndex, setRatings }) {
   const ratingOptions = [<option key="empty_option" />].concat(
     ["1", "2", "3", "4", "5"].map((option_label, index) => {
       return (
@@ -56,24 +32,37 @@ function FinalSurvey({ taskConfig, onMessageSend, active, currentCheckboxes}) {
     })
   );
 
-  const ratingSelector = (
-    <FormGroup key={"final_survey"}>
+  function handleRatingSelection(val) {
+    console.log('Old ratings: ')
+    console.log(ratings)
+    const newRatings = ratings.map((item, index) => {
+      if (index === ratingIndex) {
+        return val;
+      } else {
+        return item;
+      }
+    });
+    console.log('New ratings: ')
+    console.log(newRatings)
+    // TODO: remove all logging
+    setRatings(newRatings);
+  }
+
+  return (
+    <FormGroup key={"final_survey_" + ratingIndex.toString()}>
       <Col
         componentClass={ControlLabel}
         sm={6}
-        style={{ fontSize: "16px" }}
+        style={{ fontSize: "14px" }}
       >
-        {taskConfig.final_rating_question}
+        {ratingQuestion}
       </Col>
       <Col sm={5}>
         <FormControl
           componentClass="select"
-          style={{ fontSize: "16px" }}
-          value={rating}
-          onChange={(e) => {
-            var val = e.target.value;
-            setRating(val);
-          }}
+          style={{ fontSize: "14px" }}
+          value={ratings[ratingIndex]}
+          onChange={(e) => handleRatingSelection(e.target.value)}
           disabled={!active || sending}
         >
           {ratingOptions}
@@ -81,6 +70,61 @@ function FinalSurvey({ taskConfig, onMessageSend, active, currentCheckboxes}) {
       </Col>
     </FormGroup>
   );
+}
+
+function FinalSurvey({ taskConfig, onMessageSend, active, currentCheckboxes }) {
+  const [sending, setSending] = React.useState(false);
+
+  // Set up multiple response questions
+  let ratingQuestions = taskConfig.final_rating_question.split("|");
+  let initialRatings = [];
+  for (let _ of ratingQuestions) {
+    initialRatings.push("");
+  }
+  const [ratings, setRatings] = React.useState(initialRatings)
+
+  const tryMessageSend = React.useCallback(() => {
+
+    let all_ratings_filled = true;
+    let rating = "";
+    for (let i = 0; i < ratings.length; i++) {
+      if (ratings[i] === "") {
+        all_ratings_filled = false;
+      }
+      if (i === 0) {
+        rating += ratings[i];
+      } else {
+        rating += "|" + ratings[i];
+      }
+    }
+
+    if (all_ratings_filled && active && !sending) {
+      setSending(true);
+      onMessageSend({
+        text: "",
+        task_data: {
+          problem_data_for_prior_message: currentCheckboxes,
+          final_rating: rating,
+        },
+      }).then(() => {
+        setSending(false);
+      });
+    }
+  }, [active, sending, ratings, onMessageSend]);
+
+  const listRatingSelectors = ratingQuestions.map((ratingQuestion, ratingIndex) => {
+    return (
+      <RatingSelector
+        active={active}
+        ratings={ratings}
+        sending={sending}
+        ratingQuestion={ratingQuestion}
+        ratingIndex={ratingIndex}
+        setRatings={setRatings}
+      >
+      </RatingSelector>
+    );
+  });
 
   return (
     <div className="response-type-module">
@@ -89,22 +133,24 @@ function FinalSurvey({ taskConfig, onMessageSend, active, currentCheckboxes}) {
         the following, and hit Done.
       </div>
       <br />
-      <div className="response-bar">
-        {ratingSelector}
+      <Form
+        horizontal
+      >
+        {listRatingSelectors}
         <Button
           className="btn btn-submit submit-response"
           id="id_send_msg_button"
-          disabled={rating === 0 || !active || sending}
+          disabled={!active || sending}
           onClick={() => tryMessageSend()}
         >
           Done
         </Button>
-      </div>
+      </Form>
     </div>
   );
 }
 
-function CheckboxTextResponse({ onMessageSend, active, currentCheckboxes}) {
+function CheckboxTextResponse({ onMessageSend, active, currentCheckboxes }) {
   const [textValue, setTextValue] = React.useState("");
   const [sending, setSending] = React.useState(false);
 
@@ -119,9 +165,9 @@ function CheckboxTextResponse({ onMessageSend, active, currentCheckboxes}) {
   const tryMessageSend = React.useCallback(() => {
     if (textValue !== "" && active && !sending) {
       setSending(true);
-      onMessageSend({ 
-        text: textValue, 
-        task_data: {problem_data_for_prior_message: currentCheckboxes} 
+      onMessageSend({
+        text: textValue,
+        task_data: { problem_data_for_prior_message: currentCheckboxes }
       }).then(() => {
         setTextValue("");
         setSending(false);
@@ -169,15 +215,15 @@ function CheckboxTextResponse({ onMessageSend, active, currentCheckboxes}) {
 }
 
 function ResponseComponent({ taskConfig, appSettings, onMessageSend, active }) {
-  
+
   const lastMessageIdx = appSettings.numMessages - 1;
   const lastMessageAnnotations = appSettings.checkboxValues[lastMessageIdx];
-  
+
   const computedActive = (
-    taskConfig.annotation_buckets === null || 
+    taskConfig.annotation_buckets === null ||
     hasAnyAnnotations(lastMessageAnnotations) & active
   );
-  
+
   if (lastMessageIdx >= taskConfig.min_num_turns * 2) {
     return (
       <FinalSurvey
@@ -189,7 +235,7 @@ function ResponseComponent({ taskConfig, appSettings, onMessageSend, active }) {
     );
   } else {
     return (
-      <CheckboxTextResponse 
+      <CheckboxTextResponse
         onMessageSend={onMessageSend}
         active={computedActive}
         currentCheckboxes={lastMessageAnnotations}
diff --git a/parlai/crowdsourcing/tasks/model_chat/model_chat_blueprint.py b/parlai/crowdsourcing/tasks/model_chat/model_chat_blueprint.py
index 0c944d772ac..155ca477a3d 100644
--- a/parlai/crowdsourcing/tasks/model_chat/model_chat_blueprint.py
+++ b/parlai/crowdsourcing/tasks/model_chat/model_chat_blueprint.py
@@ -137,7 +137,12 @@ class BaseModelChatBlueprintArgs(ParlAIChatBlueprintArgs):
     )
     final_rating_question: str = field(
         default='Please rate your partner on a scale of 1-5.',
-        metadata={"help": "Text to show when asking worker to make their final rating"},
+        metadata={
+            "help": (
+                "Text to show when asking worker to make their final rating. For "
+                "multiple final ratings, separate text strings with a '|'."
+            )
+        },
     )
     max_concurrent_responses: int = field(
         default=1,

From b9e7f1fb9626de0c3a9085cd12077764f0f12f81 Mon Sep 17 00:00:00 2001
From: Eric Smith <EricMichaelSmith@users.noreply.github.com>
Date: Thu, 23 Dec 2021 17:23:37 -0500
Subject: [PATCH 2/6] Docs and cleanup

---
 parlai/crowdsourcing/tasks/model_chat/README.md              | 2 ++
 .../tasks/model_chat/frontend/components/response_panes.jsx  | 5 -----
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/parlai/crowdsourcing/tasks/model_chat/README.md b/parlai/crowdsourcing/tasks/model_chat/README.md
index 875c4c5341c..befd684421c 100644
--- a/parlai/crowdsourcing/tasks/model_chat/README.md
+++ b/parlai/crowdsourcing/tasks/model_chat/README.md
@@ -12,6 +12,8 @@ Set `mephisto.blueprint.model_opt_path` to specify a path to a YAML file listing
 
 Set `mephisto.blueprint.chat_data_folder` to the root folder that you want all results of HITs to be saved in: all results will be saved to a date folder (of format `2021_01_15`) within that root folder.
 
+Set `mephisto.blueprint.final_rating_question` to specify the question to ask the worker at the end of the task, for which the worker will respond with a 1-to-5 Likert score. Separate multiple questions with a `|`.
+
 ## Passing in task config files
 
 The following flags can be passed in to specify filepaths for overriding the text shown to the workers and the settings of the annotation categories. If they are not specified, the defaults in the `task_config/` folder will be used.
diff --git a/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx b/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx
index cc71ab44286..406263c58e1 100644
--- a/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx
+++ b/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx
@@ -33,8 +33,6 @@ function RatingSelector({ active, ratings, sending, ratingQuestion, ratingIndex,
   );
 
   function handleRatingSelection(val) {
-    console.log('Old ratings: ')
-    console.log(ratings)
     const newRatings = ratings.map((item, index) => {
       if (index === ratingIndex) {
         return val;
@@ -42,9 +40,6 @@ function RatingSelector({ active, ratings, sending, ratingQuestion, ratingIndex,
         return item;
       }
     });
-    console.log('New ratings: ')
-    console.log(newRatings)
-    // TODO: remove all logging
     setRatings(newRatings);
   }
 

From cd5c162be4d3ee46751743c95e3245b022d744bc Mon Sep 17 00:00:00 2001
From: Eric Smith <EricMichaelSmith@users.noreply.github.com>
Date: Thu, 23 Dec 2021 17:48:25 -0500
Subject: [PATCH 3/6] Show original UI if only one rating

---
 .../frontend/components/response_panes.jsx    | 36 +++++++++++++++----
 1 file changed, 29 insertions(+), 7 deletions(-)

diff --git a/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx b/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx
index 406263c58e1..67bf2f44e5c 100644
--- a/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx
+++ b/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx
@@ -121,13 +121,9 @@ function FinalSurvey({ taskConfig, onMessageSend, active, currentCheckboxes }) {
     );
   });
 
-  return (
-    <div className="response-type-module">
-      <div>
-        You've completed the conversation. Please annotate the final turn, fill out
-        the following, and hit Done.
-      </div>
-      <br />
+  if (listRatingSelectors.length > 1) {
+    // Show ratings to the right of the questions
+    const form = (
       <Form
         horizontal
       >
@@ -141,6 +137,32 @@ function FinalSurvey({ taskConfig, onMessageSend, active, currentCheckboxes }) {
           Done
         </Button>
       </Form>
+    );
+  } else {
+    // Show the single rating below the single question
+    const form = (
+      <div className="response-bar">
+        {listRatingSelectors}
+        <Button
+          className="btn btn-submit submit-response"
+          id="id_send_msg_button"
+          disabled={!active || sending}
+          onClick={() => tryMessageSend()}
+        >
+          Done
+        </Button>
+      </div>
+    );
+  }
+
+  return (
+    <div className="response-type-module">
+      <div>
+        You've completed the conversation. Please annotate the final turn, fill out
+        the following, and hit Done.
+      </div>
+      <br />
+      {form}
     </div>
   );
 }

From bc1d81933503b65b24dada6f72e350d542d8b184 Mon Sep 17 00:00:00 2001
From: EricMichaelSmith <ems@fb.com>
Date: Thu, 23 Dec 2021 18:16:48 -0500
Subject: [PATCH 4/6] Fixes

---
 .../frontend/components/response_panes.jsx    | 73 ++++++++++---------
 1 file changed, 38 insertions(+), 35 deletions(-)

diff --git a/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx b/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx
index 67bf2f44e5c..6996e0daf20 100644
--- a/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx
+++ b/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx
@@ -123,48 +123,51 @@ function FinalSurvey({ taskConfig, onMessageSend, active, currentCheckboxes }) {
 
   if (listRatingSelectors.length > 1) {
     // Show ratings to the right of the questions
-    const form = (
-      <Form
-        horizontal
-      >
-        {listRatingSelectors}
-        <Button
-          className="btn btn-submit submit-response"
-          id="id_send_msg_button"
-          disabled={!active || sending}
-          onClick={() => tryMessageSend()}
+    return (
+      <div className="response-type-module">
+        <div>
+          You've completed the conversation. Please annotate the final turn, fill out
+          the following, and hit Done.
+        </div>
+        <br />
+        <Form
+          horizontal
         >
-          Done
-        </Button>
-      </Form>
+          {listRatingSelectors}
+          <Button
+            className="btn btn-submit submit-response"
+            id="id_send_msg_button"
+            disabled={!active || sending}
+            onClick={() => tryMessageSend()}
+          >
+            Done
+          </Button>
+        </Form>
+      </div>
     );
   } else {
     // Show the single rating below the single question
-    const form = (
-      <div className="response-bar">
-        {listRatingSelectors}
-        <Button
-          className="btn btn-submit submit-response"
-          id="id_send_msg_button"
-          disabled={!active || sending}
-          onClick={() => tryMessageSend()}
-        >
-          Done
-        </Button>
+    return (
+      <div className="response-type-module">
+        <div>
+          You've completed the conversation. Please annotate the final turn, fill out
+          the following, and hit Done.
+        </div>
+        <br />
+        <div className="response-bar">
+          {listRatingSelectors}
+          <Button
+            className="btn btn-submit submit-response"
+            id="id_send_msg_button"
+            disabled={!active || sending}
+            onClick={() => tryMessageSend()}
+          >
+            Done
+          </Button>
+        </div>
       </div>
     );
   }
-
-  return (
-    <div className="response-type-module">
-      <div>
-        You've completed the conversation. Please annotate the final turn, fill out
-        the following, and hit Done.
-      </div>
-      <br />
-      {form}
-    </div>
-  );
 }
 
 function CheckboxTextResponse({ onMessageSend, active, currentCheckboxes }) {

From 9769a8dd9e93347f4913bbd55a0462dadb18b36a Mon Sep 17 00:00:00 2001
From: Eric Smith <EricMichaelSmith@users.noreply.github.com>
Date: Wed, 5 Jan 2022 17:00:25 -0500
Subject: [PATCH 5/6] Simplification

---
 .../frontend/components/response_panes.jsx         | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx b/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx
index 6996e0daf20..802dcde27e5 100644
--- a/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx
+++ b/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx
@@ -80,18 +80,8 @@ function FinalSurvey({ taskConfig, onMessageSend, active, currentCheckboxes }) {
 
   const tryMessageSend = React.useCallback(() => {
 
-    let all_ratings_filled = true;
-    let rating = "";
-    for (let i = 0; i < ratings.length; i++) {
-      if (ratings[i] === "") {
-        all_ratings_filled = false;
-      }
-      if (i === 0) {
-        rating += ratings[i];
-      } else {
-        rating += "|" + ratings[i];
-      }
-    }
+    let all_ratings_filled = ratings.every((r) => r !== "");
+    let rating = ratings.join('|');
 
     if (all_ratings_filled && active && !sending) {
       setSending(true);

From 88a402733a4f2ae3bc2ba1a5cad08887f524562f Mon Sep 17 00:00:00 2001
From: EricMichaelSmith <ems@fb.com>
Date: Wed, 5 Jan 2022 17:29:15 -0500
Subject: [PATCH 6/6] Dummy