update mm_mapworld

games/mm_mapworld/instancegenerator.py

@@ -16,8 +16,9 @@
 IMAGE_PATH = os.path.join('games', 'mm_mapworld', 'resources', 'images')
 MOVE_CONSTRUCTION = "GO: "
 STOP_CONSTRUCTION = "DONE"
-DONE_REGEX = "DONE"
-MOVE_REGEX = "GO:\s*(north|east|south|west)"
+RESONSE_REGEX = '{"description":\s*".+",(\s|\n)*"action":\s*".+"}'
+DONE_REGEX = 'DONE'
+MOVE_REGEX = 'GO:\s*(north|east|west|south)'
 
 
 def create_instances(grid_size = GRIDS['medium'], graph_size = SIZES['medium'], num_instances = NUM_INSTANCES):
@@ -53,21 +54,17 @@ def instance_from_args(args, prompts):
         num_instances=NUM_INSTANCES
     )
     for i in range(len(instances)):
-        if args['images']:
-            instances[i]['initial_prompt'] = prompts['initial']
-            instances[i]['success_response'] = prompts['later_success']
-            instances[i]['invalid_response'] = prompts['later_invalid']
+        if args['one_shot']:
+            instances[i]['initial_prompt'] = prompts['initial_one_shot']
         else:
-            instances[i]['initial_prompt'] = prompts['initial_no_img']
-            instances[i]['success_response'] = prompts['later_success_no_img']
-            instances[i]['invalid_response'] = prompts['later_invalid_no_img']
-            instances[i]['use_images'] = False
+            instances[i]['initial_prompt'] = prompts['initial']
+        instances[i]['success_response'] = prompts['later_success']
+        instances[i]['invalid_response'] = prompts['later_invalid']
         if args['reprompt']:
             instances[i]['reprompt'] = True
         instances[i]["reprompt_format"] = prompts["reprompt_format"]
         instances[i]["limit_warning"] = prompts["limit_warning"]
         instances[i]["loop_warning"] = prompts["loop_warning"]
-
     return instances
 
 
@@ -79,28 +76,22 @@ def __init__(self):
     def on_generate(self):
         prompts = {
             'initial': self.load_template('resources/initial_prompts/prompt.template'),
-            'initial_no_img': self.load_template('resources/initial_prompts/prompt_no_img.template'),
+            'initial_one_shot': self.load_template('resources/initial_prompts/prompt_one_shot.template'),
             'later_success': self.load_template('resources/later_prompts/successful_move.template'),
             'later_invalid': self.load_template('resources/later_prompts/invalid_move.template'),
-            'later_success_no_img': self.load_template('resources/later_prompts/successful_move_no_img.template'),
-            'later_invalid_no_img': self.load_template('resources/later_prompts/invalid_move_no_img.template'),
             'reprompt_format': self.load_template('resources/reprompts/invalid_format.template'),
             'limit_warning': self.load_template('resources/later_prompts/turn_limit.template'),
             'loop_warning': self.load_template('resources/later_prompts/loop.template'),
         }
         experiments = {
-            'random_small': {"size": "small", "images": True, "reprompt": False},
-            'random_medium': {"size": "medium", "images": True, "reprompt": False},
-            # 'random_large': {"size": "large", "images": True, "reprompt": False},
-            # 'random_small_reprompt': {"size": "small", "images": True, "reprompt": True},
-#             'random_medium_reprompt': {"size": "medium", "images": True, "reprompt": True},
-            # 'random_large_reprompt': {"size": "large", "images": True, "reprompt": True},
-            # 'random_small_no_img': {"size": "small", "images": False, "reprompt": False},
-            'random_medium_no_img': {"size": "medium", "images": False, "reprompt": False},
-            # 'random_large_no_img': {"size": "large", "images": False, "reprompt": False},
-            # 'random_small_no_img_reprompt': {"size": "small", "images": False, "reprompt": True},
-#             'random_medium_no_img_reprompt': {"size": "medium", "images": False, "reprompt": True},
-            # 'random_large_no_img_reprompt': {"size": "large", "images": False, "reprompt": True}
+#             'random_small': {"size": "small", "reprompt": False},
+#             'random_medium': {"size": "medium", "reprompt": False},
+            'random_small_one_shot': {"size": "small", "reprompt": False, "one_shot": True},
+            'random_medium_one_shot': {"size": "medium", "reprompt": False, "one_shot": True},
+            # 'random_large': {"size": "large", "reprompt": False},
+            # 'random_small_reprompt': {"size": "small", "reprompt": True},
+            # 'random_medium_reprompt': {"size": "medium", "reprompt": True},
+            # 'random_large_reprompt': {"size": "large", "reprompt": True},
         }
 
         for exp in experiments.keys():
@@ -113,6 +104,7 @@ def on_generate(self):
                      instance[key] = value
                  instance["move_construction"] = MOVE_CONSTRUCTION
                  instance["stop_construction"] = STOP_CONSTRUCTION
+                 instance["response_regex"] = RESONSE_REGEX
                  instance["done_regex"] = DONE_REGEX
                  instance["move_regex"] = MOVE_REGEX
                  game_id += 1

games/mm_mapworld/master.py

@@ -91,7 +91,7 @@ def cardinal_room_change(self, cardinal):
             self.current_room = new_room
 
     def _custom_response(self, messages, turn_idx) -> str:
-        last_move = messages[-1]['content']
+        last_move = json.loads(messages[-1]['content'])['action']
         without_move = last_move.replace('GO:', '')
         words = without_move.strip().split()
         new_dir = words[0]
@@ -151,6 +151,7 @@ def _on_setup(self, **game_instance):
         self.init_prompt = game_instance["initial_prompt"]
         self.visited_nodes=[self.current_room]
 
+        self.response_regex = re.compile(game_instance["response_regex"])
         self.done_regex = re.compile(game_instance["done_regex"])
         self.move_regex = re.compile(game_instance["move_regex"])
 
@@ -187,44 +188,46 @@ def _does_game_proceed(self):
         return False
 
     def _on_parse_response(self, player: Player, utterance: str) -> Tuple[str, bool]:
-        """
-        Hook
-
-        Decide if a response utterance should be modified. If not simply return the utterance.
-
-        When a modified utterance and a true value is returned, then a 'parse' event is logged.
-
-        :param player: that produced the response
-        :param utterance: to be potentially modified
-        :return: the (modified) utterance and if to log the parse action (default: True)
-        """
         if player == self.walker:
             utterance = utterance.replace("\n", "").strip()
             for word in DIRS:
                 utterance = utterance.replace(word.capitalize(), word)
-            done_hit = re.search(self.done_regex, utterance)
-            if done_hit:
-                utterance = done_hit.group()
-            hit = re.search(self.move_regex, utterance)
-            if hit:
-                utterance = hit.group()
+                utterance = utterance.replace(word.upper(), word)
+            found = re.search(self.response_regex, utterance)
+            if found:
+                utterance = found.group()
         return utterance, True
 
     def _validate_player_response(self, player: Player, answer: str) -> bool:
-        """Check if the utterance conforms to rules (cloudgame specific)."""
-        answer = answer.replace("\n", "").strip()
-        for word in DIRS:
-            answer = answer.replace(word.capitalize(), word)
         if player == self.walker:
+            answer = answer.replace("\n", "").strip()
+            for word in DIRS:
+                answer = answer.replace(word.capitalize(), word)
+                answer = answer.replace(word.upper(), word)
             # in case we abort we set the next move to None
             self.move = None
             # Check if the answer begins with 'MOVE:'
-            done_hit = re.search(self.done_regex, answer)
-            if done_hit:
+            hit = re.search(self.response_regex, answer)
+            if not hit:
+                if self.do_reprompt:
+                    if self.did_reprompt:
+                        self.aborted = True
+                        self.log_to_self("Invalid format", "Game aborted.")
+                        return False
+                    self.need_reprompt = True
+                    self.log_to_self("reprompting", "invalid format")
+                    return True
+                self.aborted = True
+                self.log_to_self("Invalid format", "Game aborted.")
+                return False
+            loaded = json.loads(hit.group())
+            action = loaded["action"]
+            action_hit = re.search(self.done_regex, action)
+            if action_hit:
                 self.stop = True
                 self.log_to_self("DONE", True)
                 return True
-            hit = re.search(self.move_regex, answer)
+            hit = re.search(self.move_regex, action)
             if not hit:
                 if self.do_reprompt:
                     if self.did_reprompt:
@@ -240,7 +243,6 @@ def _validate_player_response(self, player: Player, answer: str) -> bool:
             new_dir = hit.group(1)
             self.move = new_dir
             self.log_to_self("Valid format", "Continue")
-
         return True
 
     def _after_add_player_response(self, player: Player, utterance: str):
@@ -286,17 +288,24 @@ def _on_after_turn(self, turn_idx: int):
 
 
     ########## Multimodal specific functions:
+
+    def remove_previous_images(self, player: Player):
+        history = self.messages_by_names[player.descriptor]
+        for i in range(len(history)-1):
+            if "image" in history[i]:
+                del history[i]['image']
 
     def add_message(self, player: Player, utterance: str, role: str, image = None):
         if image is None:
             message = {"role": role, "content": utterance}
         else:
             message = {"role": role, "content": utterance, "image": image}
+            self.remove_previous_images(player)
         history = self.messages_by_names[player.descriptor]
         history.append(message)
 
     def add_user_message(self, player: Player, utterance: str, image = None):
-        self.add_message(player, utterance, role="user", image= image)
+        self.add_message(player, utterance, role="user", image=image)
 
 
     ####### scoring      
@@ -352,7 +361,6 @@ def find_best_moves(self, current, visited):
                         q.put(new)
         return found
 
-    #BETA
     def plot_path(self, path):
         offset = 0.03
         fig = plt.figure(figsize=(4, 4))
@@ -475,31 +483,31 @@ def compute_scores(self, episode_interactions) -> None:
 
 
 
-    # def store_scores(self, results_root: str, dialogue_pair: str, game_record_dir: str):
-    #     self.store_results_file(self.scores, "scores.json",
-    #                             dialogue_pair=dialogue_pair,
-    #                             sub_dir=game_record_dir,
-    #                             root_dir=results_root)
+    def store_scores(self, results_root: str, dialogue_pair: str, game_record_dir: str):
+        self.store_results_file(self.scores, "scores.json",
+                                dialogue_pair=dialogue_pair,
+                                sub_dir=game_record_dir,
+                                root_dir=results_root)
 
-    #     # plotting & animation
-    #     if not os.path.exists("tmp"):
-    #         os.makedirs("tmp")
-    #     path_plot = self.plot_path(self.path)
-    #     path_plot.savefig(os.path.join(results_root, dialogue_pair, self.name, game_record_dir, "path.png"))
-    #     plt.close()
-    #     if not os.path.exists("tmp/step_plots"):
-    #         os.makedirs("tmp/step_plots")
-    #     images = []
-    #     for i in range(len(self.path)):
-    #         step_plot = self.plot_path(self.path[:i+1])
-    #         step_plot.savefig(f"tmp/step_plots/{i}.png")
-    #         images.append(imageio.imread(f"tmp/step_plots/{i}.png"))
-    #         plt.close()
-    #     imageio.mimsave(os.path.join(results_root, dialogue_pair, self.name, game_record_dir, "animation.gif"), images, fps=1, loop=True)
-    #     try:
-    #         shutil.rmtree("tmp")
-    #     except OSError as e:
-    #         print("Error: %s - %s." % (e.filename, e.strerror))
+        # plotting & animation
+        if not os.path.exists("tmp"):
+            os.makedirs("tmp")
+        path_plot = self.plot_path(self.path)
+        path_plot.savefig(os.path.join(results_root, dialogue_pair, self.name, game_record_dir, "path.png"))
+        plt.close()
+        if not os.path.exists("tmp/step_plots"):
+            os.makedirs("tmp/step_plots")
+        images = []
+        for i in range(len(self.path)):
+            step_plot = self.plot_path(self.path[:i+1])
+            step_plot.savefig(f"tmp/step_plots/{i}.png")
+            images.append(imageio.imread(f"tmp/step_plots/{i}.png"))
+            plt.close()
+        imageio.mimsave(os.path.join(results_root, dialogue_pair, self.name, game_record_dir, "animation.gif"), images, fps=1, loop=True)
+        try:
+            shutil.rmtree("tmp")
+        except OSError as e:
+            print("Error: %s - %s." % (e.filename, e.strerror))
 
 
 

games/mm_mapworld/resources/initial_prompts/prompt.template

@@ -1,3 +1,3 @@
-We are currently in this room. Please help me with the following task. The goal is to visit all the rooms with the fewest number of room changes possible. In each room you need to decide the direction to go in. Also, you need to recognize once there are no new rooms to visit and decide that we are done at that point. Please give your answer in the following format: “GO: DIRECTION”: to step into a new room, replace DIRECTION with one of [north, south, east, west]. “DONE”: This will end the game and we will be graded.
+We are currently in this room. Please help me with the following task. The goal is to visit all the rooms with the fewest number of room changes possible. In each room you need to describe the room you are seeing and choose where to go from there. Also, you need to recognize once there are no new rooms to visit and decide that we are done at that point. Please give your answer in the following format: "{"description": "<room description>", "action": "<action>"}". Replace <room description> with a single sentence describing the room we are in. To move to a nieghboring room, replace <action> with "GO: DIRECTION" where DIRECTION can be one of [north, south, east, west]. To stop the exploration replace <action> with "DONE".
 Let us start. 
-From this room we can go in the following directions: $INITIAL_DIRECTIONS$. What should we do?
+We are now in this room. From here we can go: $INITIAL_DIRECTIONS$. What should we do?

games/mm_mapworld/resources/initial_prompts/prompt_one_shot.template

@@ -0,0 +1,11 @@
+We are currently in this room. Please help me with the following task. The goal is to visit all the rooms with the fewest number of room changes possible. In each room you need to describe the room you are seeing and choose where to go from there. Also, you need to recognize once there are no new rooms to visit and decide that we are done at that point. Please give your answer in the following format: “{"description": "<room description>", "action": "<action>"}”. Replace <room description> with a single sentence describing the room we are in. To move to a nieghboring room, replace <action> with "GO: DIRECTION" where DIRECTION can be one of [north, south, east, west]. To stop the exploration replace <action> with "DONE".
+Here is an example:
+We are now in this room. From here we can go: north, west. What should we do?
+{"description": "We are in a kitchen with a red fridge.", "action": "GO: north"}
+We are now in this room. From here we can go: south, east. What should we do?
+{"description": "We are in a living room with a couch and a tv.", "action": "GO: east"}
+...
+We are now in this room. From here we can go: south, east. What should we do?
+{"description": "We are in a bathroom", "action": "DONE"}
+Let us start. 
+We are now in this room. From here we can go: $INITIAL_DIRECTIONS$. What should we do?

games/mm_mapworld/resources/later_prompts/invalid_move.template

@@ -1 +1 @@
-The move was invalid and we are still in this room <image>. From here we can go: $DIRECTIONS$. What should we do?
+The move was invalid and we are still in this room. From here we can go: $DIRECTIONS$. What should we do?

games/mm_mapworld/resources/later_prompts/loop.template

@@ -1 +1 @@
-It seems like we are going back and forth between two rooms. If you want to stop exploring simply respond with "DONE". 
+It seems like we are going back and forth between two rooms. To stop exploring choose "DONE" as our next action. 

games/mm_mapworld/resources/later_prompts/successful_move.template

@@ -1 +1 @@
-We are now in this room <image>. From here we can go: $DIRECTIONS$. What should we do?
+We are now in this room. From here we can go: $DIRECTIONS$. What should we do?

games/mm_mapworld/resources/later_prompts/turn_limit.template

@@ -1 +1 @@
-We have been exploring for a while now. If you think we have seen most rooms simply respond with "DONE".
+We have been exploring for a while now. To stop exploring choose "DONE" as our next action. 

games/mm_mapworld/resources/reprompts/invalid_format.template

@@ -1 +1 @@
-Please only give your answer in the given format so I can understand it. To move to a nieghboring room answer with "GO: <direction>" and replace <direction> with one of the available directions which are: $DIRECTIONS$. If you think there are no more rooms we need to visit, simply anser with "DONE". What should we do? 
+Please only give your answer in the provided format so I can understand it. The format is {"description": "<room description>", "action": "<action>"}". Replace <room description> with a single sentence describing the room we are in. To move to a nieghboring room, replace <action> with "GO: DIRECTION" where DIRECTION can be one of [north, south, east, west]. To stop the exploration replace <action> with "DONE". What should we do?