[Tests] Multiwoz determinism

parlai/tasks/multiwoz_v22/agents.py

@@ -454,7 +454,7 @@ def _get_curr_belief_states(self, turn):
                             domain, slot_type = domain_slot_type.split("-")
                             belief_state = f"{domain} {slot_type} {slot_value.lower()}"
                             belief_states.append(belief_state)
-        return list(set(belief_states))
+        return sorted(list(set(belief_states)))
 
     def _extract_slot_from_string(self, slots_string):
         """
@@ -590,9 +590,7 @@ def setup_data(self, fold):
                             "dialogue_id": dialog["dialogue_id"],
                             "turn_num": turn["turn_id"],
                             "text": " ".join(context),
-                            "labels": self.BELIEF_STATE_DELIM.join(
-                                set(cum_belief_states)
-                            ),
+                            "labels": self.BELIEF_STATE_DELIM.join(cum_belief_states),
                         }
                     )
         if not is_this_circleci():

parlai/tasks/multiwoz_v22/test/multiwoz_v22_MultiWOZv22DSTTeacher_test.yml

@@ -9,7 +9,7 @@ acts:
 - - dialogue_id: MUL0484.json
     episode_done: true
     eval_labels:
-    - train departure norwich ; train arriveby 18:00 ; train day monday ; train destination
+    - train arriveby 18:00 ; train day monday ; train departure norwich ; train destination
       cambridge
     id: MultiwozV22_UserSimulatorTeacher
     text: <user> i need train reservations from norwich to cambridge <system> i have
@@ -19,7 +19,7 @@ acts:
 - - dialogue_id: MUL0484.json
     episode_done: true
     eval_labels:
-    - train departure norwich ; train arriveby 18:00 ; train day monday ; train destination
+    - train arriveby 18:00 ; train day monday ; train departure norwich ; train destination
       cambridge
     id: MultiwozV22_UserSimulatorTeacher
     text: <user> i need train reservations from norwich to cambridge <system> i have
@@ -32,7 +32,7 @@ acts:
 - - dialogue_id: MUL0484.json
     episode_done: true
     eval_labels:
-    - train arriveby 18:00 ; train day monday ; attraction name cineworld cinema ;
+    - attraction name cineworld cinema ; train arriveby 18:00 ; train day monday ;
       train departure norwich ; train destination cambridge
     id: MultiwozV22_UserSimulatorTeacher
     text: <user> i need train reservations from norwich to cambridge <system> i have
@@ -48,7 +48,7 @@ acts:
 - - dialogue_id: MUL0484.json
     episode_done: true
     eval_labels:
-    - train arriveby 18:00 ; train day monday ; attraction name cineworld cinema ;
+    - attraction name cineworld cinema ; train arriveby 18:00 ; train day monday ;
       train departure norwich ; train destination cambridge
     id: MultiwozV22_UserSimulatorTeacher
     text: <user> i need train reservations from norwich to cambridge <system> i have

parlai/tasks/multiwoz_v22/test/multiwoz_v22_MultiWOZv22DSTTeacher_train.yml

@@ -3,14 +3,14 @@ acts:
     episode_done: true
     id: MultiwozV22_UserSimulatorTeacher
     labels:
-    - restaurant pricerange expensive ; restaurant area centre
+    - restaurant area centre ; restaurant pricerange expensive
     text: <user> i need a place to dine in the center thats expensive
     turn_num: '0'
 - - dialogue_id: PMUL4398.json
     episode_done: true
     id: MultiwozV22_UserSimulatorTeacher
     labels:
-    - restaurant pricerange expensive ; restaurant area centre
+    - restaurant area centre ; restaurant pricerange expensive
     text: <user> i need a place to dine in the center thats expensive <system> i have
       several options for you; do you prefer african, asian, or british food? <user>
       any sort of food would be fine, as long as it is a bit expensive. could i get
@@ -20,8 +20,8 @@ acts:
     episode_done: true
     id: MultiwozV22_UserSimulatorTeacher
     labels:
-    - restaurant name bedouin ; hotel type hotel ; restaurant pricerange expensive
-      ; hotel pricerange expensive ; restaurant area centre
+    - hotel pricerange expensive ; hotel type hotel ; restaurant area centre ; restaurant
+      name bedouin ; restaurant pricerange expensive
     text: <user> i need a place to dine in the center thats expensive <system> i have
       several options for you; do you prefer african, asian, or british food? <user>
       any sort of food would be fine, as long as it is a bit expensive. could i get
@@ -33,9 +33,8 @@ acts:
     episode_done: true
     id: MultiwozV22_UserSimulatorTeacher
     labels:
-    - hotel type hotel ; restaurant name bedouin ; restaurant pricerange expensive
-      ; hotel pricerange expensive ; restaurant area centre ; hotel name university
-      arms hotel
+    - hotel name university arms hotel ; hotel pricerange expensive ; hotel type hotel
+      ; restaurant area centre ; restaurant name bedouin ; restaurant pricerange expensive
     text: <user> i need a place to dine in the center thats expensive <system> i have
       several options for you; do you prefer african, asian, or british food? <user>
       any sort of food would be fine, as long as it is a bit expensive. could i get
@@ -49,9 +48,9 @@ acts:
     episode_done: true
     id: MultiwozV22_UserSimulatorTeacher
     labels:
-    - hotel bookday saturday ; hotel type hotel ; restaurant name bedouin ; restaurant
-      pricerange expensive ; hotel pricerange expensive ; restaurant area centre ;
-      hotel bookpeople 2 ; hotel bookstay 2 ; hotel name university arms hotel
+    - hotel bookday saturday ; hotel bookpeople 2 ; hotel bookstay 2 ; hotel name
+      university arms hotel ; hotel pricerange expensive ; hotel type hotel ; restaurant
+      area centre ; restaurant name bedouin ; restaurant pricerange expensive
     text: <user> i need a place to dine in the center thats expensive <system> i have
       several options for you; do you prefer african, asian, or british food? <user>
       any sort of food would be fine, as long as it is a bit expensive. could i get

parlai/tasks/multiwoz_v22/test/multiwoz_v22_MultiWOZv22DSTTeacher_valid.yml

@@ -20,8 +20,8 @@ acts:
 - - dialogue_id: PMUL0698.json
     episode_done: true
     eval_labels:
-    - restaurant area centre ; train day sunday ; train leaveat 16:15 ; restaurant
-      food chinese
+    - restaurant area centre ; restaurant food chinese ; train day sunday ; train
+      leaveat 16:15
     id: MultiwozV22_UserSimulatorTeacher
     text: <user> i'm looking for a local place to dine in the centre that serves chinese
       food. <system> i have restaurants matching your criteria in all price ranges.
@@ -34,8 +34,8 @@ acts:
 - - dialogue_id: PMUL0698.json
     episode_done: true
     eval_labels:
-    - train departure cambridge ; train day sunday ; train destination norwich ; restaurant
-      area centre ; train leaveat 16:15 ; restaurant food chinese
+    - restaurant area centre ; restaurant food chinese ; train day sunday ; train
+      departure cambridge ; train destination norwich ; train leaveat 16:15
     id: MultiwozV22_UserSimulatorTeacher
     text: <user> i'm looking for a local place to dine in the centre that serves chinese
       food. <system> i have restaurants matching your criteria in all price ranges.
@@ -50,9 +50,9 @@ acts:
 - - dialogue_id: PMUL0698.json
     episode_done: true
     eval_labels:
-    - train departure cambridge ; train day sunday ; train destination norwich ; train
-      bookpeople 5 ; restaurant area centre ; train leaveat 16:15 ; restaurant food
-      chinese
+    - restaurant area centre ; restaurant food chinese ; train bookpeople 5 ; train
+      day sunday ; train departure cambridge ; train destination norwich ; train leaveat
+      16:15
     id: MultiwozV22_UserSimulatorTeacher
     text: <user> i'm looking for a local place to dine in the centre that serves chinese
       food. <system> i have restaurants matching your criteria in all price ranges.