Standardize Output Dictionaries, Mock API Calls, Math (GSM8K, SVAMP, TabMWP) for Reflexion

agential/cog/agent/react.py

@@ -96,13 +96,17 @@ def generate(
             )
 
             # Observe.
-            obs = self.strategy.generate_observation(
+            obs, external_tool_info = self.strategy.generate_observation(
                 idx=idx, action_type=action_type, query=query
             )
 
             out.append(
                 self.strategy.create_output_dict(
-                    thought=thought, action_type=action_type, query=query, obs=obs
+                    thought=thought,
+                    action_type=action_type,
+                    query=query,
+                    obs=obs,
+                    external_tool_info=external_tool_info,
                 )
             )
 

agential/cog/functional/reflexion.py

@@ -178,13 +178,27 @@ def _prompt_cot_agent(
         prompt=prompt,
         additional_keys=additional_keys,
     )
+    print(
+        "<PROMPT AGENT========================================================================>"
+    )
+    print(prompt)
+    print(
+        "<PROMPT AGENT========================================================================>"
+    )
     out = llm(
         [
             HumanMessage(
                 content=prompt,
             )
         ]
     ).content
+    print(
+        "<OUT AGENT========================================================================>"
+    )
+    print(repr(out))
+    print(
+        "<OUT AGENT========================================================================>"
+    )
     assert isinstance(out, str)
     return out
 
@@ -252,13 +266,27 @@ def _prompt_cot_reflection(
         prompt=prompt,
         additional_keys=additional_keys,
     )
+    print(
+        "<PROMPT REFLECT========================================================================>"
+    )
+    print(prompt)
+    print(
+        "<PROMPT REFLECT========================================================================>"
+    )
     out = llm(
         [
             HumanMessage(
                 content=prompt,
             )
         ]
     ).content
+    print(
+        "<OUT REFLECT========================================================================>"
+    )
+    print(repr(out))
+    print(
+        "<OUT REFLECT========================================================================>"
+    )
     assert isinstance(out, str)
     return out
 

agential/cog/prompts/agent/reflexion.py

@@ -602,3 +602,12 @@
 Action 2: Finish[Lindsey Vonn]
 
 Reflection: My reasoning failed because I doubted the clear evidence provided by the source and made an incorrect assumption based on the fame of another skier. In the future, I should rely on the provided evidence rather than making unsupported assumptions."""
+
+
+# ======================================================================== GSM8K ======================================================================== #
+
+
+# ======================================================================== SVAMP ======================================================================== #
+
+
+# ======================================================================== TABMWP ======================================================================== #

agential/cog/prompts/benchmark/gsm8k.py

@@ -80,6 +80,104 @@
 answer = chocolates_left"""
 
 
+GSM8K_FEWSHOT_EXAMPLES_COT = """Question: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to Denny?
+Thought: Let's think step by step. Jason had 20 lollipops initially and now he has 12 lollipops. So, he must have given 20 - 12 = 8 lollipops to Denny.
+Action: Finish[
+```python
+jason_lollipops_initial = 20
+jason_lollipops_after = 12
+denny_lollipops = jason_lollipops_initial - jason_lollipops_after
+answer = denny_lollipops
+```
+]
+
+Question: There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there will be 21 trees. How many trees did the grove workers plant today?
+Thought: Let's think step by step. There were initially 15 trees and after planting, there are 21 trees. So, the number of trees planted is 21 - 15 = 6.
+Action: Finish[
+```python
+trees_initial = 15
+trees_after = 21
+trees_added = trees_after - trees_initial
+answer = trees_added
+```
+]
+
+Question: Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?
+Thought: Let's think step by step. Shawn had 5 toys initially, and he got 2 toys from his mom and 2 toys from his dad. So, the total number of toys he has now is 5 + 2 + 2 = 9.
+Action: Finish[
+```python
+toys_initial = 5
+mom_toys = 2
+dad_toys = 2
+total_received = mom_toys + dad_toys
+total_toys = toys_initial + total_received
+answer = total_toys
+```
+]
+
+Question: There were nine computers in the server room. Five more computers were installed each day, from Monday to Thursday. How many computers are now in the server room?
+Thought: Let's think step by step. Initially, there were 9 computers. Five more computers were added each day from Monday to Thursday, which is 4 days. So, the total number of computers added is 5 * 4 = 20. Therefore, the total number of computers now is 9 + 20 = 29.
+Action: Finish[
+```python
+computers_initial = 9
+computers_per_day = 5
+num_days = 4  # 4 days between Monday and Thursday
+computers_added = computers_per_day * num_days
+computers_total = computers_initial + computers_added
+answer = computers_total
+```
+]
+
+Question: Michael had 58 golf balls. On Tuesday, he lost 23 golf balls. On Wednesday, he lost 2 more. How many golf balls did he have at the end of Wednesday?
+Thought: Let's think step by step. Michael initially had 58 golf balls. He lost 23 golf balls on Tuesday and 2 more on Wednesday. So, the total number of golf balls lost is 23 + 2 = 25. Therefore, the number of golf balls he has left is 58 - 25 = 33.
+Action: Finish[
+```python
+golf_balls_initial = 58
+golf_balls_lost_tuesday = 23
+golf_balls_lost_wednesday = 2
+golf_balls_left = golf_balls_initial - golf_balls_lost_tuesday - golf_balls_lost_wednesday
+answer = golf_balls_left
+```
+]
+
+Question: If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?
+Thought: Let's think step by step. Initially, there are 3 cars in the parking lot. When 2 more cars arrive, the total number of cars is 3 + 2 = 5.
+Action: Finish[
+```python
+cars_initial = 3
+cars_arrived = 2
+total_cars = cars_initial + cars_arrived
+answer = total_cars
+```
+]
+
+Question: Olivia has $23. She bought five bagels for $3 each. How much money does she have left?
+Thought: Let's think step by step. Olivia had $23 initially. She bought 5 bagels for $3 each. The total cost of the bagels is 5 * 3 = $15. So, the money she has left is 23 - 15 = $8.
+Action: Finish[
+```python
+money_initial = 23
+bagels = 5
+bagel_cost = 3
+money_spent = bagels * bagel_cost
+money_left = money_initial - money_spent
+answer = money_left
+```
+]
+
+Question: Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?
+Thought: Let's think step by step. Leah had 32 chocolates and her sister had 42 chocolates. Together they had 32 + 42 = 74 chocolates. They ate 35 chocolates, so the number of chocolates left is 74 - 35 = 39.
+Action: Finish[
+```python
+leah_chocolates = 32
+sister_chocolates = 42
+total_chocolates = leah_chocolates + sister_chocolates
+chocolates_eaten = 35
+chocolates_left = total_chocolates - chocolates_eaten
+answer = chocolates_left
+```
+]"""
+
+
 GSM8K_FEWSHOT_EXAMPLES_REACT = """Question: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to Denny?
 Thought 1: First, I need to find out how many lollipops Jason gave to Denny.
 Action 1: Calculate[

agential/cog/strategies/react/base.py

@@ -1,7 +1,7 @@
 """Base ReAct Agent strategy class."""
 
 from abc import abstractmethod
-from typing import Dict, Tuple
+from typing import Any, Dict, Tuple
 
 from langchain_core.language_models.chat_models import BaseChatModel
 
@@ -41,7 +41,9 @@ def generate_action(
         pass
 
     @abstractmethod
-    def generate_observation(self, idx: int, action_type: str, query: str) -> str:
+    def generate_observation(
+        self, idx: int, action_type: str, query: str
+    ) -> Tuple[str, Dict[str, Any]]:
         """Generates an observation based on the action type and query.
 
         Args:
@@ -50,13 +52,18 @@ def generate_observation(self, idx: int, action_type: str, query: str) -> str:
             query (str): The query for the action.
 
         Returns:
-            str: The generated observation.
+            Tuple[str, Dict[str, Any]]: The generated observation and external tool outputs.
         """
         pass
 
     @abstractmethod
     def create_output_dict(
-        self, thought: str, action_type: str, query: str, obs: str
+        self,
+        thought: str,
+        action_type: str,
+        query: str,
+        obs: str,
+        external_tool_info: Dict[str, Any],
     ) -> Dict[str, str]:
         """Creates a dictionary of the output components.
 
@@ -65,9 +72,10 @@ def create_output_dict(
             action_type (str): The type of action performed.
             query (str): The query for the action.
             obs (str): The generated observation.
+            external_tool_info (Dict[str, Any]): The external tool outputs.
 
         Returns:
-            Dict[str, str]: A dictionary containing the thought, action type, query, and observation.
+            Dict[str, Any]: A dictionary containing the thought, action type, query, observation, answer, and external tool output.
         """
         pass
 

agential/cog/strategies/react/code.py

@@ -66,7 +66,7 @@ def __init__(
         self.enc = enc
 
         self._scratchpad = ""
-        self._current_answer = ""
+        self._answer = ""
         self._finished = False
 
     def generate(
@@ -148,7 +148,9 @@ def generate_action(
 
         return action_type, query
 
-    def generate_observation(self, idx: int, action_type: str, query: str) -> str:
+    def generate_observation(
+        self, idx: int, action_type: str, query: str
+    ) -> Tuple[str, Dict[str, Any]]:
         """Generates an observation based on the action type and query.
 
         Args:
@@ -157,47 +159,63 @@ def generate_observation(self, idx: int, action_type: str, query: str) -> str:
             query (str): The query for the action.
 
         Returns:
-            str: The generated observation.
+            Tuple[str, Dict[str, Any]]: The generated observation and external tool outputs.
         """
+        external_tool_info = {"execution_status": ""}
+
         self._scratchpad += f"\nObservation {idx}: "
         if action_type.lower() == "finish":
-            self._current_answer = query
+            _, execution_status = safe_execute(query)
+            external_tool_info["execution_status"] = execution_status
+
+            self._answer = query
             self._finished = True
-            obs = f"\n```python\n{self._current_answer}\n```"
+            obs = f"\n```python\n{self._answer}\n```"
         elif action_type.lower() == "implement":
             _, execution_status = safe_execute(query)
-            self._current_answer = query
-            obs = f"\n```python\n{self._current_answer}\n```\nExecution Status: {execution_status}"
+            external_tool_info["execution_status"] = execution_status
+
+            self._answer = query
+            obs = f"\n```python\n{self._answer}\n```\nExecution Status: {execution_status}"
         elif action_type.lower() == "test":
-            obs = f"{self._current_answer}\n\n{query}"
+            obs = f"{self._answer}\n\n{query}"
             _, execution_status = safe_execute(obs)
+            external_tool_info["execution_status"] = execution_status
+
             obs = f"\n```python\n{obs}\n```\nExecution Status: {execution_status}"
         else:
             obs = "Invalid Action. Valid Actions are Implement[code] Test[code] and Finish[answer]."
         self._scratchpad += obs
 
-        return obs
+        return obs, external_tool_info
 
     def create_output_dict(
-        self, thought: str, action_type: str, query: str, obs: str
-    ) -> Dict[str, str]:
+        self,
+        thought: str,
+        action_type: str,
+        query: str,
+        obs: str,
+        external_tool_info: Dict[str, Any],
+    ) -> Dict[str, Any]:
         """Creates a dictionary of the output components.
 
         Args:
             thought (str): The generated thought.
             action_type (str): The type of action performed.
             query (str): The query for the action.
             obs (str): The generated observation.
+            external_tool_info (Dict[str, Any]): The external tool outputs.
 
         Returns:
-            Dict[str, str]: A dictionary containing the thought, action type, query, observation, and answer.
+            Dict[str, Any]: A dictionary containing the thought, action type, query, observation, answer, and external tool output.
         """
         return {
             "thought": thought,
             "action_type": action_type,
             "query": query,
             "observation": obs,
-            "answer": self._current_answer,
+            "answer": self._answer,
+            "external_tool_info": external_tool_info,
         }
 
     def halting_condition(
@@ -248,7 +266,7 @@ def reset(self, **kwargs: Any) -> None:
         Returns:
             None
         """
-        self._current_answer = ""
+        self._answer = ""
         self._scratchpad = ""
         self._finished = False