From 67228105f435a9fe5686240acb7ecf1d9cbc6f3d Mon Sep 17 00:00:00 2001 From: Boqi Chen Date: Thu, 26 Oct 2023 16:35:44 -0400 Subject: [PATCH 01/18] Add flag parser --- .../{config.py => config/__init__.py} | 6 +++ src/sherpa_ai/config/task_config.py | 43 +++++++++++++++++++ src/tests/unit_tests/test_config.py | 31 +++++++++++++ 3 files changed, 80 insertions(+) rename src/sherpa_ai/{config.py => config/__init__.py} (97%) create mode 100644 src/sherpa_ai/config/task_config.py create mode 100644 src/tests/unit_tests/test_config.py diff --git a/src/sherpa_ai/config.py b/src/sherpa_ai/config/__init__.py similarity index 97% rename from src/sherpa_ai/config.py rename to src/sherpa_ai/config/__init__.py index 28c2c0aa..bca23579 100644 --- a/src/sherpa_ai/config.py +++ b/src/sherpa_ai/config/__init__.py @@ -17,6 +17,8 @@ from dotenv import find_dotenv, load_dotenv from loguru import logger +from sherpa_ai.config.task_config import AgentConfig + env_path = find_dotenv(usecwd=True) load_dotenv(env_path) @@ -106,3 +108,7 @@ def check_vectordb_setting(): logger.info("Config: OpenAI environment variables are set") check_vectordb_setting() + +__all__ = [ + "AgentConfig", +] diff --git a/src/sherpa_ai/config/task_config.py b/src/sherpa_ai/config/task_config.py new file mode 100644 index 00000000..386e6a8e --- /dev/null +++ b/src/sherpa_ai/config/task_config.py @@ -0,0 +1,43 @@ +import re +from argparse import ArgumentParser +from typing import List, Optional + +from pydantic import BaseModel + + +class AgentConfig(BaseModel): + verbose: bool = False + verbosex: bool = False + gsite: Optional[str] = None + do_reflect: bool = False + + @classmethod + def from_input(cls, input_str: str) -> "AgentConfig": + """ + parse input string into AgentConfig. The configurations are + at the end of the string + """ + parts = re.split(r"(?=--)", input_str) + configs = [] + + for part in parts[1:]: + part = part.strip() + configs.extend(part.split()) + + return cls.from_config(configs) + + @classmethod + def from_config(cls, configs: List[str]) -> "AgentConfig": + parser = ArgumentParser() + + parser.add_argument("--verbose", action="store_true") + parser.add_argument("--verbosex", action="store_true") + parser.add_argument("--gsite", type=str, default=None) + parser.add_argument("--do-reflect", action="store_true") + + args, unknown = parser.parse_known_args(configs) + + if len(unknown) > 0: + raise ValueError(f"Invalid configuration, check your input: {unknown}") + + return AgentConfig(**args.__dict__) diff --git a/src/tests/unit_tests/test_config.py b/src/tests/unit_tests/test_config.py new file mode 100644 index 00000000..658eaa47 --- /dev/null +++ b/src/tests/unit_tests/test_config.py @@ -0,0 +1,31 @@ +import pytest + +from sherpa_ai.config import AgentConfig + + +def test_parse_args(): + site = "https://www.google.com" + input_str = f"Test input. --verbose --verbosex --gsite {site}" + + config = AgentConfig.from_input(input_str) + + assert config.verbose + assert config.verbosex + assert config.gsite == site + + +def test_parse_args_partial(): + input_str = "Test input. --verbose" + + config = AgentConfig.from_input(input_str) + + assert config.verbose + assert config.gsite is None + + +def test_parse_args_noise(): + site = "https://www.google.com" + input_str = f"This is an input with -- but--should not be considered --verbose --verbosex --gsite {site} --do-reflect" # noqa: E501 + + with pytest.raises(ValueError): + AgentConfig.from_input(input_str) From b6549c8a4071cc4b8ae7672cb801fa95e035e2cc Mon Sep 17 00:00:00 2001 From: Boqi Chen Date: Thu, 26 Oct 2023 23:53:26 -0400 Subject: [PATCH 02/18] Refactor slack app and add more tests --- src/apps/slackapp/slackapp/bolt_app.py | 83 +++++++++----------- src/apps/slackapp/tests/test_get_response.py | 55 +++++++++++++ src/sherpa_ai/config/task_config.py | 8 +- src/sherpa_ai/task_agent.py | 53 ++++++------- src/tests/unit_tests/test_config.py | 9 ++- 5 files changed, 124 insertions(+), 84 deletions(-) create mode 100644 src/apps/slackapp/tests/test_get_response.py diff --git a/src/apps/slackapp/slackapp/bolt_app.py b/src/apps/slackapp/slackapp/bolt_app.py index 5a7983ae..1ac62f0b 100644 --- a/src/apps/slackapp/slackapp/bolt_app.py +++ b/src/apps/slackapp/slackapp/bolt_app.py @@ -7,12 +7,14 @@ from typing import Dict, List from flask import Flask, request +from langchain.schema import AIMessage, BaseMessage, HumanMessage from loguru import logger from slack_bolt import App from slack_bolt.adapter.flask import SlackRequestHandler from slackapp.routes.whitelist import whitelist_blueprint import sherpa_ai.config as cfg +from sherpa_ai.config import AgentConfig from sherpa_ai.connectors.vectorstores import get_vectordb from sherpa_ai.database.user_usage_tracker import UserUsageTracker from sherpa_ai.error_handling import AgentErrorHandler @@ -46,22 +48,32 @@ def hello_command(ack, body): ack(f"Hi, <@{user_id}>!") -def contains_verbose(query: str) -> bool: - """looks for -verbose in the question and returns True or False""" - return "-verbose" in query.lower() +def process_chat_history(self, messages: List[dict]) -> List[BaseMessage]: + results = [] + for message in messages: + logger.info(message) + if message["type"] != "message" and message["type"] != "text": + continue -def contains_verbosex(query: str) -> bool: - """looks for -verbosex in the question and returns True or False""" - return "-verbosex" in query.lower() + message_cls = AIMessage if message["user"] == self.ai_id else HumanMessage + # replace the at in the message with the name of the bot + text = message["text"].replace(f"@{self.ai_id}", f"@{self.ai_name}") + # added by JF + text = text.split("#verbose", 1)[0] # remove everything after #verbose + text = text.replace("-verbose", "") # remove -verbose if it exists + results.append(message_cls(content=text)) + + return results def get_response( question: str, - previous_messages: List[Dict], - verbose_logger: BaseVerboseLogger, + previous_messages: List[BaseMessage], user_id: str, team_id: str, + verbose_logger: BaseVerboseLogger, + bot_dict: Dict[str, str] ): llm = SherpaChatOpenAI( openai_api_key=cfg.OPENAI_API_KEY, @@ -73,14 +85,18 @@ def get_response( memory = get_vectordb() + question, agent_config = AgentConfig.from_input(question) + # check if the verbose is on + verbose_logger = verbose_logger if agent_config.verbose else DummyVerboseLogger() + tools = get_tools(memory) ai_name = "Sherpa" - ai_id = bot["user_id"] + ai_id = bot_dict["user_id"] task_agent = TaskAgent.from_llm_and_tools( ai_name="Sherpa", ai_role="assistant", - ai_id=bot["user_id"], + ai_id=bot_dict["user_id"], memory=memory, tools=tools, previous_messages=previous_messages, @@ -90,33 +106,9 @@ def get_response( error_handler = AgentErrorHandler() question = question.replace(f"@{ai_id}", f"@{ai_name}") - if contains_verbosex(query=question): - logger.info("Verbose mode is on, show all") - question = question.replace("-verbose", "") - response = error_handler.run_with_error_handling(task_agent.run, task=question) - agent_log = task_agent.logger # logger is updated after running task_agent.run - try: # in case log_formatter fails - verbose_message = log_formatter(agent_log) - except KeyError: - verbose_message = str(agent_log) - return response, verbose_message - - elif contains_verbose(query=question): - logger.info("Verbose mode is on, commands only") - question = question.replace("-verbose", "") - response = error_handler.run_with_error_handling(task_agent.run, task=question) - - agent_log = task_agent.logger # logger is updated after running task_agent.run - try: # in case log_formatter fails - verbose_message = show_commands_only(agent_log) - except KeyError: - verbose_message = str(agent_log) - return response, verbose_message + response = error_handler.run_with_error_handling(task_agent.run, task=question) - else: - logger.info("Verbose mode is off") - response = error_handler.run_with_error_handling(task_agent.run, task=question) - return response, None + return response @app.event("app_mention") @@ -125,12 +117,7 @@ def event_test(client, say, event): thread_ts = event.get("thread_ts", None) or event["ts"] replies = client.conversations_replies(channel=event["channel"], ts=thread_ts) previous_messages = replies["messages"][:-1] - - # check if the verbose is on - verbose_on = contains_verbose(question) - verbose_logger = ( - SlackVerboseLogger(say, thread_ts) if verbose_on else DummyVerboseLogger() - ) + previous_messages = process_chat_history(previous_messages) input_message = replies["messages"][-1] user_id = input_message["user"] @@ -162,12 +149,20 @@ def event_test(client, say, event): ) question = reconstructor.reconstruct_prompt() results, _ = get_response( - question, previous_messages, verbose_logger, user_id, team_id + question, + previous_messages, + user_id, + team_id, + verbose_logger=SlackVerboseLogger(say, thread_ts), + bot_dict=bot, ) say(results, thread_ts=thread_ts) else: - say(f"""I'm sorry for any inconvenience, but it appears you've gone over your daily token limit. Don't worry, you'll be able to use our service again in approximately {usage_cheker['time_left']}.Thank you for your patience and understanding.""", thread_ts=thread_ts) + say( + f"""I'm sorry for any inconvenience, but it appears you've gone over your daily token limit. Don't worry, you'll be able to use our service again in approximately {usage_cheker['time_left']}.Thank you for your patience and understanding.""", + thread_ts=thread_ts, + ) @app.event("app_home_opened") diff --git a/src/apps/slackapp/tests/test_get_response.py b/src/apps/slackapp/tests/test_get_response.py new file mode 100644 index 00000000..1f67d2d8 --- /dev/null +++ b/src/apps/slackapp/tests/test_get_response.py @@ -0,0 +1,55 @@ +from datetime import datetime + +import pytest +from slackapp.bolt_app import get_response + +import sherpa_ai.config as cfg +from sherpa_ai.verbose_loggers import DummyVerboseLogger + + +def test_get_date(): + question = "What is the date today, using the following format: YYYY-MM-DD?" + date = datetime.now().strftime("%Y-%m-%d") + + if cfg.SERPER_API_KEY is None: + pytest.skip( + "SERPER_API_KEY not found in environment variables, skipping this test" + ) + + verbose_logger = DummyVerboseLogger() + + response = get_response( + question=question, + previous_messages=[], + user_id="", + team_id="", + verbose_logger=verbose_logger, + bot_dict={"user_id": "Sherpa"}, + ) + assert date in response, "Today's date not found in response" + + +def test_get_question(): + question = "What is AutoGPT and how does it compare with MetaGPT" + + if cfg.SERPER_API_KEY is None: + pytest.skip( + "SERPER_API_KEY not found in environment variables, skipping this test" + ) + + verbose_logger = DummyVerboseLogger() + + response = get_response( + question=question, + previous_messages=[], + user_id="", + team_id="", + verbose_logger=verbose_logger, + bot_dict={"user_id": "Sherpa"}, + ) + + print(response) + assert response is not None + assert response != "" + assert "AutoGPT" in response + assert "MetaGPT" in response diff --git a/src/sherpa_ai/config/task_config.py b/src/sherpa_ai/config/task_config.py index 386e6a8e..9f5745a4 100644 --- a/src/sherpa_ai/config/task_config.py +++ b/src/sherpa_ai/config/task_config.py @@ -1,18 +1,17 @@ import re from argparse import ArgumentParser -from typing import List, Optional +from typing import List, Optional, Tuple from pydantic import BaseModel class AgentConfig(BaseModel): verbose: bool = False - verbosex: bool = False gsite: Optional[str] = None do_reflect: bool = False @classmethod - def from_input(cls, input_str: str) -> "AgentConfig": + def from_input(cls, input_str: str) -> Tuple[str, "AgentConfig"]: """ parse input string into AgentConfig. The configurations are at the end of the string @@ -24,14 +23,13 @@ def from_input(cls, input_str: str) -> "AgentConfig": part = part.strip() configs.extend(part.split()) - return cls.from_config(configs) + return parts[0].strip(), cls.from_config(configs) @classmethod def from_config(cls, configs: List[str]) -> "AgentConfig": parser = ArgumentParser() parser.add_argument("--verbose", action="store_true") - parser.add_argument("--verbosex", action="store_true") parser.add_argument("--gsite", type=str, default=None) parser.add_argument("--do-reflect", action="store_true") diff --git a/src/sherpa_ai/task_agent.py b/src/sherpa_ai/task_agent.py index 765620ba..949c4262 100644 --- a/src/sherpa_ai/task_agent.py +++ b/src/sherpa_ai/task_agent.py @@ -21,6 +21,7 @@ from sherpa_ai.action_planner import SelectiveActionPlanner from sherpa_ai.action_planner.base import BaseActionPlanner +from sherpa_ai.config import AgentConfig from sherpa_ai.output_parser import BaseTaskOutputParser, TaskOutputParser from sherpa_ai.output_parsers import LinkParser, MDToSlackParse from sherpa_ai.post_processors import md_link_to_slack @@ -42,14 +43,14 @@ def __init__( action_planner: BaseActionPlanner, output_parser: BaseTaskOutputParser, tools: List[BaseTool], - previous_messages: List[dict], + previous_messages: List[BaseMessage], verbose_logger: BaseVerboseLogger, feedback_tool: Optional[HumanInputRun] = None, + agent_config: AgentConfig = AgentConfig(), max_iterations: int = 5, ): self.ai_name = ai_name self.memory = memory - # self.full_message_history: List[BaseMessage] = [] self.next_action_count = 0 self.llm = llm self.output_parser = output_parser @@ -61,7 +62,8 @@ def __init__( self.max_iterations = max_iterations self.loop_count = 0 self.ai_id = ai_id - self.previous_message = self.process_chat_history(previous_messages) + self.agent_config = agent_config + self.previous_message = previous_messages self.logger = [] # added by JF link_parser = LinkParser() @@ -69,8 +71,6 @@ def __init__( self.tool_output_parsers = [link_parser] self.output_parsers = [link_parser, slack_link_paerser] - # print(self.full_message_history) - # print("message:", self.previous_message) @classmethod def from_llm_and_tools( @@ -86,8 +86,10 @@ def from_llm_and_tools( human_in_the_loop: bool = False, output_parser: Optional[BaseTaskOutputParser] = None, max_iterations: int = 1, + agent_config: AgentConfig = AgentConfig(), verbose_logger: BaseVerboseLogger = DummyVerboseLogger(), ): + print(tools) if action_planner is None: action_planner = SelectiveActionPlanner( llm, tools, ai_name=ai_name, ai_role=ai_role @@ -105,6 +107,7 @@ def from_llm_and_tools( previous_messages, verbose_logger, feedback_tool=human_feedback_tool, + agent_config=agent_config, max_iterations=max_iterations, ) @@ -130,8 +133,8 @@ def run(self, task: str) -> str: "Use the above information to respond to the user's message:" f"\n{task}\n\n" "If you use any resource, then create inline citation by adding " - " of the reference document at the end of the sentence in the format " - "of 'Sentence [DocID]'\n" + " of the reference document at the end of the sentence in the " + "format of 'Sentence [DocID]'\n" "Example:\n" "Sentence1 [1]. Sentence2. Sentence3 [2].\n" "Only use the reference document. DO NOT use any links" @@ -152,13 +155,19 @@ def run(self, task: str) -> str: logger_step["reply"] = reply_json except json.JSONDecodeError: logger_step["reply"] = assistant_reply # last reply is a string + if self.agent_config.verbose: + self.verbose_logger.log(f"```{assistant_reply}```") + self.logger.append(logger_step) - ########## Serial Verbose Feature ####### - try: - formatted_logger_step = show_commands_only(logger_step) - except Exception as e: - logger.error(e) + # Serial Verbose Feature + if not self.agent_config.verbose: + try: + formatted_logger_step = show_commands_only(logger_step) + except Exception as e: + logger.error(e) + formatted_logger_step = logger_step + else: formatted_logger_step = logger_step logger.info(f"```{formatted_logger_step}```") @@ -179,7 +188,7 @@ def run(self, task: str) -> str: ): result = result["command"]["args"]["response"] except json.JSONDecodeError: - result = assistant_reply + result = str(assistant_reply) for output_parser in self.output_parsers: result = output_parser.parse_output(result) @@ -251,24 +260,6 @@ def set_user_input(self, user_input: str): self.memory.add_documents([Document(page_content=memory_to_add)]) - def process_chat_history(self, messages: List[dict]) -> List[BaseMessage]: - results = [] - - for message in messages: - logger.info(message) - if message["type"] != "message" and message["type"] != "text": - continue - - message_cls = AIMessage if message["user"] == self.ai_id else HumanMessage - # replace the at in the message with the name of the bot - text = message["text"].replace(f"@{self.ai_id}", f"@{self.ai_name}") - # added by JF - text = text.split("#verbose", 1)[0] # remove everything after #verbose - text = text.replace("-verbose", "") # remove -verbose if it exists - results.append(message_cls(content=text)) - - return results - def process_output(self, output: str) -> str: """ Process the output of the AI to remove the bot's name and replace it with @bot diff --git a/src/tests/unit_tests/test_config.py b/src/tests/unit_tests/test_config.py index 658eaa47..a1fe7c08 100644 --- a/src/tests/unit_tests/test_config.py +++ b/src/tests/unit_tests/test_config.py @@ -5,20 +5,21 @@ def test_parse_args(): site = "https://www.google.com" - input_str = f"Test input. --verbose --verbosex --gsite {site}" + input_str = f"Test input. --verbose --gsite {site}" - config = AgentConfig.from_input(input_str) + parsed, config = AgentConfig.from_input(input_str) + assert parsed == "Test input." assert config.verbose - assert config.verbosex assert config.gsite == site def test_parse_args_partial(): input_str = "Test input. --verbose" - config = AgentConfig.from_input(input_str) + parsed, config = AgentConfig.from_input(input_str) + assert parsed == "Test input." assert config.verbose assert config.gsite is None From ce9b21580937668a7a27c1cd33577a31a55b9267 Mon Sep 17 00:00:00 2001 From: Boqi Chen Date: Thu, 26 Oct 2023 23:57:42 -0400 Subject: [PATCH 03/18] Add agent_config in task agent --- src/apps/slackapp/slackapp/bolt_app.py | 1 + src/sherpa_ai/task_agent.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/slackapp/slackapp/bolt_app.py b/src/apps/slackapp/slackapp/bolt_app.py index 1ac62f0b..d468c494 100644 --- a/src/apps/slackapp/slackapp/bolt_app.py +++ b/src/apps/slackapp/slackapp/bolt_app.py @@ -102,6 +102,7 @@ def get_response( previous_messages=previous_messages, llm=llm, verbose_logger=verbose_logger, + agent_config=agent_config, ) error_handler = AgentErrorHandler() diff --git a/src/sherpa_ai/task_agent.py b/src/sherpa_ai/task_agent.py index 949c4262..3552bd43 100644 --- a/src/sherpa_ai/task_agent.py +++ b/src/sherpa_ai/task_agent.py @@ -89,7 +89,6 @@ def from_llm_and_tools( agent_config: AgentConfig = AgentConfig(), verbose_logger: BaseVerboseLogger = DummyVerboseLogger(), ): - print(tools) if action_planner is None: action_planner = SelectiveActionPlanner( llm, tools, ai_name=ai_name, ai_role=ai_role From c4a1ff1de123b089d07819fa5d93854c78fccab4 Mon Sep 17 00:00:00 2001 From: YujingYang Date: Tue, 31 Oct 2023 21:56:50 -0400 Subject: [PATCH 04/18] add gsite in the query --- src/sherpa_ai/actions/google_search.py | 11 ++++++++++ src/tests/unit_tests/test_config.py | 29 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/sherpa_ai/actions/google_search.py b/src/sherpa_ai/actions/google_search.py index 36af9266..3cdcfb05 100644 --- a/src/sherpa_ai/actions/google_search.py +++ b/src/sherpa_ai/actions/google_search.py @@ -3,6 +3,7 @@ from sherpa_ai.actions.base import BaseAction from sherpa_ai.tools import SearchTool +from sherpa_ai.config.task_config import AgentConfig SEARCH_SUMMARY_DESCRIPTION = """Role Description: {role_description} Task: {task} @@ -22,6 +23,7 @@ def __init__( role_description: str, task: str, llm: BaseLanguageModel, + config: AgentConfig, description: str = SEARCH_SUMMARY_DESCRIPTION, n: int = 5, ): @@ -31,10 +33,19 @@ def __init__( self.description = description self.llm = llm self.n = n + self.config = config self.search_tool = SearchTool() + + def config_gsite_query(self, query) -> str: + # check if the gsite is none + if self.config.gsite: + query = query + " " + self.config.gsite + return query + def execute(self, query) -> str: + query = self.config_gsite_query(query) result = self.search_tool._run(query) logger.debug("Search Result: {}", result) diff --git a/src/tests/unit_tests/test_config.py b/src/tests/unit_tests/test_config.py index a1fe7c08..ed376016 100644 --- a/src/tests/unit_tests/test_config.py +++ b/src/tests/unit_tests/test_config.py @@ -1,6 +1,11 @@ import pytest from sherpa_ai.config import AgentConfig +from sherpa_ai.tools import ContextTool, SearchTool +from langchain import GoogleSerperAPIWrapper +from sherpa_ai.actions.google_search import GoogleSearch +from langchain.llms import OpenAI +import sherpa_ai.config as cfg def test_parse_args(): @@ -30,3 +35,27 @@ def test_parse_args_noise(): with pytest.raises(ValueError): AgentConfig.from_input(input_str) + + +def test_gsite_google_search(): + site = "https://www.google.com" + input_str = f"Test input. --verbose --gsite {site}" + + question, config = AgentConfig.from_input(input_str) + + assert config.gsite == site + + role_description = "The programmer receives requirements about a program and write it" + + task = """We need to render a highly complex 3D image on the solar system. We can use any publicly avaliable + resources to achieve this task.""" # noqa: E501 + + llm = OpenAI(openai_api_key=cfg.OPENAI_API_KEY, temperature=0) + + google_search = GoogleSearch(role_description=role_description, task=task, llm=llm, config=config) + + query = "What is the weather today?" + + updated_query = google_search.config_gsite_query(query) + + assert site in updated_query From 9741e6d03baccdf70b0b737c062a82b0e20d2ddb Mon Sep 17 00:00:00 2001 From: YujingYang Date: Wed, 1 Nov 2023 09:53:00 -0400 Subject: [PATCH 05/18] fix two lines --- src/apps/slackapp/slackapp/bolt_app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apps/slackapp/slackapp/bolt_app.py b/src/apps/slackapp/slackapp/bolt_app.py index d468c494..2bebd3eb 100644 --- a/src/apps/slackapp/slackapp/bolt_app.py +++ b/src/apps/slackapp/slackapp/bolt_app.py @@ -48,7 +48,7 @@ def hello_command(ack, body): ack(f"Hi, <@{user_id}>!") -def process_chat_history(self, messages: List[dict]) -> List[BaseMessage]: +def process_chat_history(messages: List[dict]) -> List[BaseMessage]: results = [] for message in messages: @@ -149,7 +149,7 @@ def event_test(client, say, event): question=question, slack_message=[replies["messages"][-1]] ) question = reconstructor.reconstruct_prompt() - results, _ = get_response( + results = get_response( question, previous_messages, user_id, From 8c06434eda70d01c3d22a96e7783475d0f42cbcd Mon Sep 17 00:00:00 2001 From: Boqi Chen Date: Wed, 1 Nov 2023 14:58:43 -0400 Subject: [PATCH 06/18] Refactor unit tests structure --- .../{ => actions}/test_context_search.py | 0 .../unit_tests/{ => actions}/test_planning.py | 0 .../{ => agents}/test_critic_agent.py | 0 .../{ => agents}/test_ml_engineer.py | 0 .../unit_tests/{ => agents}/test_physicist.py | 0 .../unit_tests/{ => agents}/test_planner.py | 0 .../unit_tests/{ => agents}/test_qa_agent.py | 0 .../test_task_config.py} | 32 +++++++++++-------- .../test_extract_github_readme.py | 0 .../{ => scrape}/test_prompt_reconstructor.py | 0 10 files changed, 18 insertions(+), 14 deletions(-) rename src/tests/unit_tests/{ => actions}/test_context_search.py (100%) rename src/tests/unit_tests/{ => actions}/test_planning.py (100%) rename src/tests/unit_tests/{ => agents}/test_critic_agent.py (100%) rename src/tests/unit_tests/{ => agents}/test_ml_engineer.py (100%) rename src/tests/unit_tests/{ => agents}/test_physicist.py (100%) rename src/tests/unit_tests/{ => agents}/test_planner.py (100%) rename src/tests/unit_tests/{ => agents}/test_qa_agent.py (100%) rename src/tests/unit_tests/{test_config.py => config/test_task_config.py} (81%) rename src/tests/unit_tests/{ => scrape}/test_extract_github_readme.py (100%) rename src/tests/unit_tests/{ => scrape}/test_prompt_reconstructor.py (100%) diff --git a/src/tests/unit_tests/test_context_search.py b/src/tests/unit_tests/actions/test_context_search.py similarity index 100% rename from src/tests/unit_tests/test_context_search.py rename to src/tests/unit_tests/actions/test_context_search.py diff --git a/src/tests/unit_tests/test_planning.py b/src/tests/unit_tests/actions/test_planning.py similarity index 100% rename from src/tests/unit_tests/test_planning.py rename to src/tests/unit_tests/actions/test_planning.py diff --git a/src/tests/unit_tests/test_critic_agent.py b/src/tests/unit_tests/agents/test_critic_agent.py similarity index 100% rename from src/tests/unit_tests/test_critic_agent.py rename to src/tests/unit_tests/agents/test_critic_agent.py diff --git a/src/tests/unit_tests/test_ml_engineer.py b/src/tests/unit_tests/agents/test_ml_engineer.py similarity index 100% rename from src/tests/unit_tests/test_ml_engineer.py rename to src/tests/unit_tests/agents/test_ml_engineer.py diff --git a/src/tests/unit_tests/test_physicist.py b/src/tests/unit_tests/agents/test_physicist.py similarity index 100% rename from src/tests/unit_tests/test_physicist.py rename to src/tests/unit_tests/agents/test_physicist.py diff --git a/src/tests/unit_tests/test_planner.py b/src/tests/unit_tests/agents/test_planner.py similarity index 100% rename from src/tests/unit_tests/test_planner.py rename to src/tests/unit_tests/agents/test_planner.py diff --git a/src/tests/unit_tests/test_qa_agent.py b/src/tests/unit_tests/agents/test_qa_agent.py similarity index 100% rename from src/tests/unit_tests/test_qa_agent.py rename to src/tests/unit_tests/agents/test_qa_agent.py diff --git a/src/tests/unit_tests/test_config.py b/src/tests/unit_tests/config/test_task_config.py similarity index 81% rename from src/tests/unit_tests/test_config.py rename to src/tests/unit_tests/config/test_task_config.py index ed376016..a9d7b704 100644 --- a/src/tests/unit_tests/test_config.py +++ b/src/tests/unit_tests/config/test_task_config.py @@ -1,14 +1,14 @@ import pytest - -from sherpa_ai.config import AgentConfig -from sherpa_ai.tools import ContextTool, SearchTool from langchain import GoogleSerperAPIWrapper -from sherpa_ai.actions.google_search import GoogleSearch from langchain.llms import OpenAI + import sherpa_ai.config as cfg +from sherpa_ai.actions.google_search import GoogleSearch +from sherpa_ai.config import AgentConfig +from sherpa_ai.tools import ContextTool, SearchTool -def test_parse_args(): +def test_all_parameters_parse_successfully(): site = "https://www.google.com" input_str = f"Test input. --verbose --gsite {site}" @@ -19,7 +19,7 @@ def test_parse_args(): assert config.gsite == site -def test_parse_args_partial(): +def test_no_gsite_parses_successfully(): input_str = "Test input. --verbose" parsed, config = AgentConfig.from_input(input_str) @@ -37,7 +37,7 @@ def test_parse_args_noise(): AgentConfig.from_input(input_str) -def test_gsite_google_search(): +def test_invalid_input_raises_value_error(): site = "https://www.google.com" input_str = f"Test input. --verbose --gsite {site}" @@ -45,17 +45,21 @@ def test_gsite_google_search(): assert config.gsite == site - role_description = "The programmer receives requirements about a program and write it" - + role_description = ( + "The programmer receives requirements about a program and write it" + ) + task = """We need to render a highly complex 3D image on the solar system. We can use any publicly avaliable resources to achieve this task.""" # noqa: E501 - + llm = OpenAI(openai_api_key=cfg.OPENAI_API_KEY, temperature=0) - - google_search = GoogleSearch(role_description=role_description, task=task, llm=llm, config=config) + + google_search = GoogleSearch( + role_description=role_description, task=task, llm=llm, config=config + ) query = "What is the weather today?" - + updated_query = google_search.config_gsite_query(query) - + assert site in updated_query diff --git a/src/tests/unit_tests/test_extract_github_readme.py b/src/tests/unit_tests/scrape/test_extract_github_readme.py similarity index 100% rename from src/tests/unit_tests/test_extract_github_readme.py rename to src/tests/unit_tests/scrape/test_extract_github_readme.py diff --git a/src/tests/unit_tests/test_prompt_reconstructor.py b/src/tests/unit_tests/scrape/test_prompt_reconstructor.py similarity index 100% rename from src/tests/unit_tests/test_prompt_reconstructor.py rename to src/tests/unit_tests/scrape/test_prompt_reconstructor.py From 439ac7aaebde32d3c7c782324aa565b9b21e90fb Mon Sep 17 00:00:00 2001 From: YujingYang Date: Thu, 2 Nov 2023 15:24:45 -0400 Subject: [PATCH 07/18] add config to search tool --- src/sherpa_ai/tools.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/sherpa_ai/tools.py b/src/sherpa_ai/tools.py index 8b26ccce..0f3b853d 100644 --- a/src/sherpa_ai/tools.py +++ b/src/sherpa_ai/tools.py @@ -16,16 +16,17 @@ from typing_extensions import Literal import sherpa_ai.config as cfg +from sherpa_ai.config.task_config import AgentConfig -def get_tools(memory): +def get_tools(memory, config): tools = [] # tools.append(ContextTool(memory=memory)) tools.append(UserInputTool()) if cfg.SERPER_API_KEY is not None: - search_tool = SearchTool() + search_tool = SearchTool(config=config) tools.append(search_tool) else: logger.warning( @@ -82,7 +83,18 @@ class SearchTool(BaseTool): "you cannot find the information using internal search." ) + def __init__(self, config: AgentConfig): + self.config = config + + def _add_gsite(self, query: str) -> str: + # check if the gsite is none + if self.config.gsite: + query = query + " " + self.config.gsite + return query + def _run(self, query: str) -> str: + query = self._add_gsite(query, self.config) + logger.debug(f"Search query: {query}") google_serper = GoogleSerperAPIWrapper() search_results = google_serper._google_serper_api_results(query) From 47f68e238717bb0e2a6c0f1dd0f53ed6a813dfed Mon Sep 17 00:00:00 2001 From: YujingYang Date: Mon, 6 Nov 2023 10:31:34 -0500 Subject: [PATCH 08/18] add integration test and fix the error --- src/apps/slackapp/slackapp/bolt_app.py | 2 +- src/sherpa_ai/tools.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/apps/slackapp/slackapp/bolt_app.py b/src/apps/slackapp/slackapp/bolt_app.py index 2bebd3eb..9a28ca12 100644 --- a/src/apps/slackapp/slackapp/bolt_app.py +++ b/src/apps/slackapp/slackapp/bolt_app.py @@ -89,7 +89,7 @@ def get_response( # check if the verbose is on verbose_logger = verbose_logger if agent_config.verbose else DummyVerboseLogger() - tools = get_tools(memory) + tools = get_tools(memory, agent_config) ai_name = "Sherpa" ai_id = bot_dict["user_id"] diff --git a/src/sherpa_ai/tools.py b/src/sherpa_ai/tools.py index 0f3b853d..250a1745 100644 --- a/src/sherpa_ai/tools.py +++ b/src/sherpa_ai/tools.py @@ -78,14 +78,12 @@ def _arun(self, query: str) -> str: class SearchTool(BaseTool): name = "Search" + config: AgentConfig = AgentConfig() description = ( "Access the internet to search for the information. Only use this tool when " "you cannot find the information using internal search." ) - def __init__(self, config: AgentConfig): - self.config = config - def _add_gsite(self, query: str) -> str: # check if the gsite is none if self.config.gsite: From b6eaba5f6f525514cae601d63762dfd1ba2e17bb Mon Sep 17 00:00:00 2001 From: Boqi Chen Date: Tue, 7 Nov 2023 17:26:22 -0500 Subject: [PATCH 09/18] Clean up code --- src/apps/slackapp/slackapp/bolt_app.py | 32 +++++++++++++------ src/apps/slackapp/tests/test_get_response.py | 10 +++--- src/sherpa_ai/config/task_config.py | 19 +++++++++-- src/sherpa_ai/task_agent.py | 10 +++--- src/sherpa_ai/utils.py | 2 +- .../unit_tests/config/test_task_config.py | 3 +- 6 files changed, 53 insertions(+), 23 deletions(-) diff --git a/src/apps/slackapp/slackapp/bolt_app.py b/src/apps/slackapp/slackapp/bolt_app.py index 9a28ca12..e1e8bfa6 100644 --- a/src/apps/slackapp/slackapp/bolt_app.py +++ b/src/apps/slackapp/slackapp/bolt_app.py @@ -48,7 +48,7 @@ def hello_command(ack, body): ack(f"Hi, <@{user_id}>!") -def process_chat_history(messages: List[dict]) -> List[BaseMessage]: +def convert_thread_history_messages(messages: List[dict]) -> List[BaseMessage]: results = [] for message in messages: @@ -59,7 +59,7 @@ def process_chat_history(messages: List[dict]) -> List[BaseMessage]: message_cls = AIMessage if message["user"] == self.ai_id else HumanMessage # replace the at in the message with the name of the bot text = message["text"].replace(f"@{self.ai_id}", f"@{self.ai_name}") - # added by JF + text = text.split("#verbose", 1)[0] # remove everything after #verbose text = text.replace("-verbose", "") # remove -verbose if it exists results.append(message_cls(content=text)) @@ -73,8 +73,23 @@ def get_response( user_id: str, team_id: str, verbose_logger: BaseVerboseLogger, - bot_dict: Dict[str, str] -): + bot_info: Dict[str, str] +) -> str: + """ + Get response from the task agent for the question + + Args: + question (str): question to be answered + previous_messages (List[BaseMessage]): previous messages in the thread + user_id (str): user id of the user who asked the question + team_id (str): team id of the workspace + verbose_logger (BaseVerboseLogger): verbose logger to be used + bot_info (Dict[str, str]): information of the Slack bot + + Returns: + str: response from the task agent + """ + llm = SherpaChatOpenAI( openai_api_key=cfg.OPENAI_API_KEY, request_timeout=120, @@ -86,17 +101,16 @@ def get_response( memory = get_vectordb() question, agent_config = AgentConfig.from_input(question) - # check if the verbose is on verbose_logger = verbose_logger if agent_config.verbose else DummyVerboseLogger() tools = get_tools(memory, agent_config) ai_name = "Sherpa" - ai_id = bot_dict["user_id"] + ai_id = bot_info["user_id"] task_agent = TaskAgent.from_llm_and_tools( ai_name="Sherpa", ai_role="assistant", - ai_id=bot_dict["user_id"], + ai_id=bot_info["user_id"], memory=memory, tools=tools, previous_messages=previous_messages, @@ -118,7 +132,7 @@ def event_test(client, say, event): thread_ts = event.get("thread_ts", None) or event["ts"] replies = client.conversations_replies(channel=event["channel"], ts=thread_ts) previous_messages = replies["messages"][:-1] - previous_messages = process_chat_history(previous_messages) + previous_messages = convert_thread_history_messages(previous_messages) input_message = replies["messages"][-1] user_id = input_message["user"] @@ -155,7 +169,7 @@ def event_test(client, say, event): user_id, team_id, verbose_logger=SlackVerboseLogger(say, thread_ts), - bot_dict=bot, + bot_info=bot, ) say(results, thread_ts=thread_ts) diff --git a/src/apps/slackapp/tests/test_get_response.py b/src/apps/slackapp/tests/test_get_response.py index 1f67d2d8..49948c66 100644 --- a/src/apps/slackapp/tests/test_get_response.py +++ b/src/apps/slackapp/tests/test_get_response.py @@ -7,7 +7,8 @@ from sherpa_ai.verbose_loggers import DummyVerboseLogger -def test_get_date(): +@pytest.mark.real +def test_get_response_contains_todays_date(): question = "What is the date today, using the following format: YYYY-MM-DD?" date = datetime.now().strftime("%Y-%m-%d") @@ -24,12 +25,13 @@ def test_get_date(): user_id="", team_id="", verbose_logger=verbose_logger, - bot_dict={"user_id": "Sherpa"}, + bot_info={"user_id": "Sherpa"}, ) assert date in response, "Today's date not found in response" -def test_get_question(): +@pytest.mark.real +def test_response_contains_correct_info(): question = "What is AutoGPT and how does it compare with MetaGPT" if cfg.SERPER_API_KEY is None: @@ -45,7 +47,7 @@ def test_get_question(): user_id="", team_id="", verbose_logger=verbose_logger, - bot_dict={"user_id": "Sherpa"}, + bot_info={"user_id": "Sherpa"}, ) print(response) diff --git a/src/sherpa_ai/config/task_config.py b/src/sherpa_ai/config/task_config.py index 9f5745a4..3d24c6f7 100644 --- a/src/sherpa_ai/config/task_config.py +++ b/src/sherpa_ai/config/task_config.py @@ -29,9 +29,22 @@ def from_input(cls, input_str: str) -> Tuple[str, "AgentConfig"]: def from_config(cls, configs: List[str]) -> "AgentConfig": parser = ArgumentParser() - parser.add_argument("--verbose", action="store_true") - parser.add_argument("--gsite", type=str, default=None) - parser.add_argument("--do-reflect", action="store_true") + parser.add_argument( + "--verbose", + action="store_true", + help="enable verbose messaging during agent execution", + ) + parser.add_argument( + "--gsite", + type=str, + default=None, + helo="site to be used for the Google search tool.", + ) + parser.add_argument( + "--do-reflect", + action="store_true", + help="enable performing the reflection step for each agent.", + ) args, unknown = parser.parse_known_args(configs) diff --git a/src/sherpa_ai/task_agent.py b/src/sherpa_ai/task_agent.py index 3552bd43..78ecc726 100644 --- a/src/sherpa_ai/task_agent.py +++ b/src/sherpa_ai/task_agent.py @@ -64,7 +64,7 @@ def __init__( self.ai_id = ai_id self.agent_config = agent_config self.previous_message = previous_messages - self.logger = [] # added by JF + self.logger = [] link_parser = LinkParser() slack_link_paerser = MDToSlackParse() @@ -160,14 +160,14 @@ def run(self, task: str) -> str: self.logger.append(logger_step) # Serial Verbose Feature - if not self.agent_config.verbose: + if self.agent_config.verbose: + formatted_logger_step = logger_step + else: try: formatted_logger_step = show_commands_only(logger_step) - except Exception as e: + except KeyError as e: logger.error(e) formatted_logger_step = logger_step - else: - formatted_logger_step = logger_step logger.info(f"```{formatted_logger_step}```") self.verbose_logger.log(f"```{formatted_logger_step}```") diff --git a/src/sherpa_ai/utils.py b/src/sherpa_ai/utils.py index 2215687e..a9d1fe04 100644 --- a/src/sherpa_ai/utils.py +++ b/src/sherpa_ai/utils.py @@ -195,7 +195,7 @@ def show_commands_only(logs): log_strings.append(formatted_reply) else: # for final response - formatted_reply = f"""đź’ˇThought process finished!""" + formatted_reply = """đź’ˇThought process finished!""" log_strings.append(formatted_reply) log_string = "\n".join(log_strings) diff --git a/src/tests/unit_tests/config/test_task_config.py b/src/tests/unit_tests/config/test_task_config.py index a9d7b704..3d7fe731 100644 --- a/src/tests/unit_tests/config/test_task_config.py +++ b/src/tests/unit_tests/config/test_task_config.py @@ -10,13 +10,14 @@ def test_all_parameters_parse_successfully(): site = "https://www.google.com" - input_str = f"Test input. --verbose --gsite {site}" + input_str = f"Test input. --verbose --gsite {site} --do_reflect" parsed, config = AgentConfig.from_input(input_str) assert parsed == "Test input." assert config.verbose assert config.gsite == site + assert config.do_reflect def test_no_gsite_parses_successfully(): From f1a00eab47f2023710044e79c80bb1381949c88b Mon Sep 17 00:00:00 2001 From: YujingYang Date: Thu, 9 Nov 2023 16:59:50 -0500 Subject: [PATCH 10/18] make changes according to the comment --- src/sherpa_ai/config/task_config.py | 2 +- src/sherpa_ai/tools.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/sherpa_ai/config/task_config.py b/src/sherpa_ai/config/task_config.py index 3d24c6f7..f3a89c70 100644 --- a/src/sherpa_ai/config/task_config.py +++ b/src/sherpa_ai/config/task_config.py @@ -38,7 +38,7 @@ def from_config(cls, configs: List[str]) -> "AgentConfig": "--gsite", type=str, default=None, - helo="site to be used for the Google search tool.", + help="site to be used for the Google search tool.", ) parser.add_argument( "--do-reflect", diff --git a/src/sherpa_ai/tools.py b/src/sherpa_ai/tools.py index 250a1745..69d6066d 100644 --- a/src/sherpa_ai/tools.py +++ b/src/sherpa_ai/tools.py @@ -84,14 +84,11 @@ class SearchTool(BaseTool): "you cannot find the information using internal search." ) - def _add_gsite(self, query: str) -> str: - # check if the gsite is none - if self.config.gsite: - query = query + " " + self.config.gsite - return query + def query_with_gsite(self, query: str) -> str: + return query + " " + self.config.gsite if self.config.gsite else query def _run(self, query: str) -> str: - query = self._add_gsite(query, self.config) + query = self.query_with_gsite(query, self.config) logger.debug(f"Search query: {query}") google_serper = GoogleSerperAPIWrapper() From c2efe8c1158e756672d70a7abcfb2736d8cbec39 Mon Sep 17 00:00:00 2001 From: Boqi Chen Date: Tue, 14 Nov 2023 16:15:16 -0500 Subject: [PATCH 11/18] Fix issues in config google search and corresponding tests --- src/apps/slackapp/pyproject.toml | 3 ++ src/apps/slackapp/tests/test_get_response.py | 4 +-- src/pyproject.toml | 2 +- src/sherpa_ai/actions/google_search.py | 7 ++-- .../integration_tests/test_task_agent.py | 4 +-- .../unit_tests/actions/test_google_search.py | 20 +++++++++++ .../unit_tests/config/test_task_config.py | 33 ------------------- 7 files changed, 31 insertions(+), 42 deletions(-) create mode 100644 src/tests/unit_tests/actions/test_google_search.py diff --git a/src/apps/slackapp/pyproject.toml b/src/apps/slackapp/pyproject.toml index 6239107e..b43d3b30 100644 --- a/src/apps/slackapp/pyproject.toml +++ b/src/apps/slackapp/pyproject.toml @@ -33,6 +33,9 @@ build-backend = "poetry.core.masonry.api" pythonpath = [ "." ] +markers = [ + "external_api: this test calls 3rd party APIs" +] [tool.black] line-length = 88 diff --git a/src/apps/slackapp/tests/test_get_response.py b/src/apps/slackapp/tests/test_get_response.py index 49948c66..11321fef 100644 --- a/src/apps/slackapp/tests/test_get_response.py +++ b/src/apps/slackapp/tests/test_get_response.py @@ -7,7 +7,7 @@ from sherpa_ai.verbose_loggers import DummyVerboseLogger -@pytest.mark.real +@pytest.mark.external_api def test_get_response_contains_todays_date(): question = "What is the date today, using the following format: YYYY-MM-DD?" date = datetime.now().strftime("%Y-%m-%d") @@ -30,7 +30,7 @@ def test_get_response_contains_todays_date(): assert date in response, "Today's date not found in response" -@pytest.mark.real +@pytest.mark.external_api def test_response_contains_correct_info(): question = "What is AutoGPT and how does it compare with MetaGPT" diff --git a/src/pyproject.toml b/src/pyproject.toml index 9eae2d04..cc14c6c2 100644 --- a/src/pyproject.toml +++ b/src/pyproject.toml @@ -43,7 +43,7 @@ pythonpath = [ "." ] markers = [ - "real: this test calls 3rd party APIs" + "external_api: this test calls 3rd party APIs" ] [tool.black] diff --git a/src/sherpa_ai/actions/google_search.py b/src/sherpa_ai/actions/google_search.py index 3cdcfb05..e8a16b17 100644 --- a/src/sherpa_ai/actions/google_search.py +++ b/src/sherpa_ai/actions/google_search.py @@ -2,8 +2,8 @@ from loguru import logger from sherpa_ai.actions.base import BaseAction -from sherpa_ai.tools import SearchTool from sherpa_ai.config.task_config import AgentConfig +from sherpa_ai.tools import SearchTool SEARCH_SUMMARY_DESCRIPTION = """Role Description: {role_description} Task: {task} @@ -36,13 +36,12 @@ def __init__( self.config = config self.search_tool = SearchTool() - + def config_gsite_query(self, query) -> str: # check if the gsite is none if self.config.gsite: - query = query + " " + self.config.gsite + query = query + " site:" + self.config.gsite return query - def execute(self, query) -> str: query = self.config_gsite_query(query) diff --git a/src/tests/integration_tests/test_task_agent.py b/src/tests/integration_tests/test_task_agent.py index 3892c505..2cdc1c91 100644 --- a/src/tests/integration_tests/test_task_agent.py +++ b/src/tests/integration_tests/test_task_agent.py @@ -31,7 +31,7 @@ def config_task_agent( return task_agent -@pytest.mark.real +@pytest.mark.external_api def test_task_solving_with_search(): """Test task solving with search""" question = "What is the date today, using the following format: YYYY-MM-DD?" @@ -50,7 +50,7 @@ def test_task_solving_with_search(): assert date in response, "Today's date not found in response" -@pytest.mark.real +@pytest.mark.external_api def test_task_solving_with_context_search(): question = "What is langchain?" diff --git a/src/tests/unit_tests/actions/test_google_search.py b/src/tests/unit_tests/actions/test_google_search.py new file mode 100644 index 00000000..e9ca85a0 --- /dev/null +++ b/src/tests/unit_tests/actions/test_google_search.py @@ -0,0 +1,20 @@ +from sherpa_ai.actions.google_search import GoogleSearch +from sherpa_ai.config import AgentConfig + + +def test_gsite_config_successfully(): + site = "https://www.google.com" + config = AgentConfig( + verbose=True, + gsite=site, + ) + + assert config.gsite == site + + google_search = GoogleSearch(role_description="", task="", llm=None, config=config) + + query = "What is the weather today?" + + updated_query = google_search.config_gsite_query(query) + + assert f"site:{site}" in updated_query diff --git a/src/tests/unit_tests/config/test_task_config.py b/src/tests/unit_tests/config/test_task_config.py index 3d7fe731..0d49f0e0 100644 --- a/src/tests/unit_tests/config/test_task_config.py +++ b/src/tests/unit_tests/config/test_task_config.py @@ -1,11 +1,6 @@ import pytest -from langchain import GoogleSerperAPIWrapper -from langchain.llms import OpenAI -import sherpa_ai.config as cfg -from sherpa_ai.actions.google_search import GoogleSearch from sherpa_ai.config import AgentConfig -from sherpa_ai.tools import ContextTool, SearchTool def test_all_parameters_parse_successfully(): @@ -36,31 +31,3 @@ def test_parse_args_noise(): with pytest.raises(ValueError): AgentConfig.from_input(input_str) - - -def test_invalid_input_raises_value_error(): - site = "https://www.google.com" - input_str = f"Test input. --verbose --gsite {site}" - - question, config = AgentConfig.from_input(input_str) - - assert config.gsite == site - - role_description = ( - "The programmer receives requirements about a program and write it" - ) - - task = """We need to render a highly complex 3D image on the solar system. We can use any publicly avaliable - resources to achieve this task.""" # noqa: E501 - - llm = OpenAI(openai_api_key=cfg.OPENAI_API_KEY, temperature=0) - - google_search = GoogleSearch( - role_description=role_description, task=task, llm=llm, config=config - ) - - query = "What is the weather today?" - - updated_query = google_search.config_gsite_query(query) - - assert site in updated_query From c8dc1acf6a1682f43b34b90335e4155f1262809f Mon Sep 17 00:00:00 2001 From: Boqi Chen Date: Wed, 15 Nov 2023 19:23:33 -0500 Subject: [PATCH 12/18] Fix failed test cases --- src/sherpa_ai/actions/google_search.py | 2 +- src/sherpa_ai/config/task_config.py | 2 +- src/sherpa_ai/tools.py | 6 ------ src/tests/integration_tests/test_task_agent.py | 2 +- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/sherpa_ai/actions/google_search.py b/src/sherpa_ai/actions/google_search.py index e8a16b17..9ba89241 100644 --- a/src/sherpa_ai/actions/google_search.py +++ b/src/sherpa_ai/actions/google_search.py @@ -23,7 +23,7 @@ def __init__( role_description: str, task: str, llm: BaseLanguageModel, - config: AgentConfig, + config: AgentConfig = AgentConfig(), description: str = SEARCH_SUMMARY_DESCRIPTION, n: int = 5, ): diff --git a/src/sherpa_ai/config/task_config.py b/src/sherpa_ai/config/task_config.py index f3a89c70..54408e60 100644 --- a/src/sherpa_ai/config/task_config.py +++ b/src/sherpa_ai/config/task_config.py @@ -41,7 +41,7 @@ def from_config(cls, configs: List[str]) -> "AgentConfig": help="site to be used for the Google search tool.", ) parser.add_argument( - "--do-reflect", + "--do_reflect", action="store_true", help="enable performing the reflection step for each agent.", ) diff --git a/src/sherpa_ai/tools.py b/src/sherpa_ai/tools.py index 69d6066d..3ebf9697 100644 --- a/src/sherpa_ai/tools.py +++ b/src/sherpa_ai/tools.py @@ -78,18 +78,12 @@ def _arun(self, query: str) -> str: class SearchTool(BaseTool): name = "Search" - config: AgentConfig = AgentConfig() description = ( "Access the internet to search for the information. Only use this tool when " "you cannot find the information using internal search." ) - def query_with_gsite(self, query: str) -> str: - return query + " " + self.config.gsite if self.config.gsite else query - def _run(self, query: str) -> str: - query = self.query_with_gsite(query, self.config) - logger.debug(f"Search query: {query}") google_serper = GoogleSerperAPIWrapper() search_results = google_serper._google_serper_api_results(query) diff --git a/src/tests/integration_tests/test_task_agent.py b/src/tests/integration_tests/test_task_agent.py index 2cdc1c91..97fefdd0 100644 --- a/src/tests/integration_tests/test_task_agent.py +++ b/src/tests/integration_tests/test_task_agent.py @@ -42,7 +42,7 @@ def test_task_solving_with_search(): "SERPER_API_KEY not found in environment variables, skipping this test" ) memory = get_vectordb() - tools = [SearchTool(api_wrapper=GoogleSerperAPIWrapper())] + tools = [SearchTool()] task_agent = config_task_agent(tools=tools, memory=memory) From 30094568c0065a96d1de15b27cde31457deaa5a5 Mon Sep 17 00:00:00 2001 From: Boqi Chen Date: Wed, 15 Nov 2023 19:46:33 -0500 Subject: [PATCH 13/18] Run two test suites separately --- src/Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index fbf20323..d152c6b0 100644 --- a/src/Makefile +++ b/src/Makefile @@ -11,4 +11,7 @@ lint: black . --check test: - pytest . \ No newline at end of file + echo "Running tests for sherpa" + pytest tests + echo "Running tests for sherpa slackapp" + cd apps/slackapp && pytest tests \ No newline at end of file From 56005b10f364d2a9ebc028018b29dbcb2c05548b Mon Sep 17 00:00:00 2001 From: Boqi Chen Date: Wed, 15 Nov 2023 19:55:29 -0500 Subject: [PATCH 14/18] Add gsite at only one place --- src/sherpa_ai/actions/google_search.py | 12 ++---------- src/sherpa_ai/tools.py | 13 +++++++++++-- src/tests/unit_tests/agents/test_planner.py | 7 ++++--- .../test_search_tool.py} | 6 +++--- 4 files changed, 20 insertions(+), 18 deletions(-) rename src/tests/unit_tests/{actions/test_google_search.py => tools/test_search_tool.py} (60%) diff --git a/src/sherpa_ai/actions/google_search.py b/src/sherpa_ai/actions/google_search.py index 9ba89241..e2f6a1c9 100644 --- a/src/sherpa_ai/actions/google_search.py +++ b/src/sherpa_ai/actions/google_search.py @@ -23,8 +23,8 @@ def __init__( role_description: str, task: str, llm: BaseLanguageModel, - config: AgentConfig = AgentConfig(), description: str = SEARCH_SUMMARY_DESCRIPTION, + config: AgentConfig = AgentConfig(), n: int = 5, ): self.role_description = role_description @@ -33,18 +33,10 @@ def __init__( self.description = description self.llm = llm self.n = n - self.config = config - - self.search_tool = SearchTool() - def config_gsite_query(self, query) -> str: - # check if the gsite is none - if self.config.gsite: - query = query + " site:" + self.config.gsite - return query + self.search_tool = SearchTool(config=config) def execute(self, query) -> str: - query = self.config_gsite_query(query) result = self.search_tool._run(query) logger.debug("Search Result: {}", result) diff --git a/src/sherpa_ai/tools.py b/src/sherpa_ai/tools.py index 3ebf9697..2f36f2d0 100644 --- a/src/sherpa_ai/tools.py +++ b/src/sherpa_ai/tools.py @@ -69,7 +69,7 @@ def _run(self, query: str) -> str: ) logger.debug(f"Arxiv Search Result: {result_list}") - + return " ".join(result_list) def _arun(self, query: str) -> str: @@ -78,12 +78,21 @@ def _arun(self, query: str) -> str: class SearchTool(BaseTool): name = "Search" + config = AgentConfig() description = ( "Access the internet to search for the information. Only use this tool when " "you cannot find the information using internal search." ) + def augment_query(self, query) -> str: + # check if the gsite is none + if self.config.gsite: + query = query + " site:" + self.config.gsite + return query + def _run(self, query: str) -> str: + query = self.augment_query(query) + logger.debug(f"Search query: {query}") google_serper = GoogleSerperAPIWrapper() search_results = google_serper._google_serper_api_results(query) @@ -191,7 +200,7 @@ def _arun(self, query: str) -> str: class UserInputTool(BaseTool): - # TODO: Make an action for the user input + # TODO: Make an action for the user input name = "UserInput" description = ( "Access the user input for the task." diff --git a/src/tests/unit_tests/agents/test_planner.py b/src/tests/unit_tests/agents/test_planner.py index a26873f3..ed331330 100644 --- a/src/tests/unit_tests/agents/test_planner.py +++ b/src/tests/unit_tests/agents/test_planner.py @@ -4,7 +4,8 @@ from sherpa_ai.agents.agent_pool import AgentPool from sherpa_ai.agents.physicist import Physicist from sherpa_ai.agents.planner import Planner -from sherpa_ai.agents.programmer import Programmer + +# from sherpa_ai.agents.programmer import Programmer from sherpa_ai.memory.shared_memory import SharedMemory @@ -12,7 +13,7 @@ def test_planner(): programmer_description = ( "The programmer receives requirements about a program and write it" ) - programmer = Programmer(name="Programmer", description=programmer_description) + # programmer = Programmer(name="Programmer", description=programmer_description) physicist_description = ( "The physicist agent answers questions or research about physics-related topics" @@ -20,7 +21,7 @@ def test_planner(): physicist = Physicist(name="Physicist", description=physicist_description) agent_pool = AgentPool() - agent_pool.add_agents([programmer, physicist]) + agent_pool.add_agents([physicist]) shared_memeory = SharedMemory( objective="Share the information across different agents.", diff --git a/src/tests/unit_tests/actions/test_google_search.py b/src/tests/unit_tests/tools/test_search_tool.py similarity index 60% rename from src/tests/unit_tests/actions/test_google_search.py rename to src/tests/unit_tests/tools/test_search_tool.py index e9ca85a0..670a7300 100644 --- a/src/tests/unit_tests/actions/test_google_search.py +++ b/src/tests/unit_tests/tools/test_search_tool.py @@ -1,5 +1,5 @@ -from sherpa_ai.actions.google_search import GoogleSearch from sherpa_ai.config import AgentConfig +from sherpa_ai.tools import SearchTool def test_gsite_config_successfully(): @@ -11,10 +11,10 @@ def test_gsite_config_successfully(): assert config.gsite == site - google_search = GoogleSearch(role_description="", task="", llm=None, config=config) + search_tool = SearchTool(config=config) query = "What is the weather today?" - updated_query = google_search.config_gsite_query(query) + updated_query = search_tool.augment_query(query) assert f"site:{site}" in updated_query From fbe562677febafc0a1b6fc23360e8a165e386468 Mon Sep 17 00:00:00 2001 From: Boqi Chen Date: Wed, 15 Nov 2023 19:58:25 -0500 Subject: [PATCH 15/18] fix grammar issue --- src/sherpa_ai/task_agent.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sherpa_ai/task_agent.py b/src/sherpa_ai/task_agent.py index 78ecc726..d349d4cf 100644 --- a/src/sherpa_ai/task_agent.py +++ b/src/sherpa_ai/task_agent.py @@ -131,9 +131,9 @@ def run(self, task: str) -> str: user_input = ( "Use the above information to respond to the user's message:" f"\n{task}\n\n" - "If you use any resource, then create inline citation by adding " - " of the reference document at the end of the sentence in the " - "format of 'Sentence [DocID]'\n" + "If you use any resource, then create inline citation by adding" + "the DocID of the reference document at the end of the sentence in " + "the format of 'Sentence [DocID]'\n" "Example:\n" "Sentence1 [1]. Sentence2. Sentence3 [2].\n" "Only use the reference document. DO NOT use any links" From 34bc7120d7da0bb3f5c6763bb0802e9e90e0c23b Mon Sep 17 00:00:00 2001 From: Oshoma Momoh Date: Thu, 16 Nov 2023 10:40:16 -0500 Subject: [PATCH 16/18] Rename test and use ternary operator --- src/sherpa_ai/tools.py | 5 +---- src/tests/unit_tests/tools/test_search_tool.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/sherpa_ai/tools.py b/src/sherpa_ai/tools.py index 2f36f2d0..3146cb5f 100644 --- a/src/sherpa_ai/tools.py +++ b/src/sherpa_ai/tools.py @@ -85,10 +85,7 @@ class SearchTool(BaseTool): ) def augment_query(self, query) -> str: - # check if the gsite is none - if self.config.gsite: - query = query + " site:" + self.config.gsite - return query + return query + " site:" + self.config.gsite if self.config.gsite else query def _run(self, query: str) -> str: query = self.augment_query(query) diff --git a/src/tests/unit_tests/tools/test_search_tool.py b/src/tests/unit_tests/tools/test_search_tool.py index 670a7300..ab78e80e 100644 --- a/src/tests/unit_tests/tools/test_search_tool.py +++ b/src/tests/unit_tests/tools/test_search_tool.py @@ -2,7 +2,7 @@ from sherpa_ai.tools import SearchTool -def test_gsite_config_successfully(): +def test_search_query_includes_gsite_config(): site = "https://www.google.com" config = AgentConfig( verbose=True, From b34e838833f19a9ded1c54e4c4cfaf255342987f Mon Sep 17 00:00:00 2001 From: Oshoma Momoh Date: Thu, 16 Nov 2023 11:09:01 -0500 Subject: [PATCH 17/18] Fix broken tests --- src/sherpa_ai/actions/planning.py | 4 ++-- src/tests/unit_tests/actions/test_planning.py | 2 -- .../unit_tests/agents/test_critic_agent.py | 4 ++-- src/tests/unit_tests/agents/test_planner.py | 20 +++++++++++-------- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/sherpa_ai/actions/planning.py b/src/sherpa_ai/actions/planning.py index 4ec6c646..e49b1c7a 100644 --- a/src/sherpa_ai/actions/planning.py +++ b/src/sherpa_ai/actions/planning.py @@ -116,8 +116,8 @@ def execute( self, task: str, agent_pool_description: str, - last_plan: Optional[str], - feedback: Optional[str], + last_plan: Optional[str] = None, + feedback: Optional[str] = None, ) -> Plan: """ Execute the action diff --git a/src/tests/unit_tests/actions/test_planning.py b/src/tests/unit_tests/actions/test_planning.py index c441ee88..ba49d967 100644 --- a/src/tests/unit_tests/actions/test_planning.py +++ b/src/tests/unit_tests/actions/test_planning.py @@ -4,8 +4,6 @@ import sherpa_ai.config as cfg from sherpa_ai.actions.planning import TaskPlanning -llm = OpenAI(openai_api_key=cfg.OPENAI_API_KEY, temperature=0) - def test_planning(): llm = OpenAI(openai_api_key=cfg.OPENAI_API_KEY, temperature=0) diff --git a/src/tests/unit_tests/agents/test_critic_agent.py b/src/tests/unit_tests/agents/test_critic_agent.py index cb3ffd95..d64bd440 100644 --- a/src/tests/unit_tests/agents/test_critic_agent.py +++ b/src/tests/unit_tests/agents/test_critic_agent.py @@ -21,7 +21,7 @@ def test_evaluation_matrices(): llm = OpenAI(openai_api_key=cfg.OPENAI_API_KEY, temperature=0) - critic_agent = Critic(name="CriticAgent", llm=llm, ratio=1) + critic_agent = Critic(llm=llm, ratio=1) i_score, i_evaluation = critic_agent.get_importance_evaluation(task, plan) assert type(i_score) is int @@ -34,7 +34,7 @@ def test_evaluation_matrices(): def test_get_feedback(): llm = OpenAI(openai_api_key=cfg.OPENAI_API_KEY, temperature=0) - critic_agent = Critic(name="CriticAgent", llm=llm, ratio=1) + critic_agent = Critic(llm=llm, ratio=1) feedback_list = critic_agent.get_feedback(task, plan) assert len(feedback_list) == 3 # assert type(feedback) is str diff --git a/src/tests/unit_tests/agents/test_planner.py b/src/tests/unit_tests/agents/test_planner.py index ed331330..83f85d6c 100644 --- a/src/tests/unit_tests/agents/test_planner.py +++ b/src/tests/unit_tests/agents/test_planner.py @@ -10,30 +10,34 @@ def test_planner(): - programmer_description = ( - "The programmer receives requirements about a program and write it" - ) + # programmer_description = ( + # "The programmer receives requirements about a program and write it" + # ) # programmer = Programmer(name="Programmer", description=programmer_description) + llm = OpenAI(openai_api_key=cfg.OPENAI_API_KEY, temperature=0) + physicist_description = ( "The physicist agent answers questions or research about physics-related topics" ) - physicist = Physicist(name="Physicist", description=physicist_description) + physicist = Physicist( + name="Physicist", + description=physicist_description, + llm=llm, + ) agent_pool = AgentPool() agent_pool.add_agents([physicist]) - shared_memeory = SharedMemory( + shared_memory = SharedMemory( objective="Share the information across different agents.", agent_pool=agent_pool, ) - llm = OpenAI(openai_api_key=cfg.OPENAI_API_KEY, temperature=0) - planner = Planner( name="planner", agent_pool=agent_pool, - shared_memory=shared_memeory, + shared_memory=shared_memory, llm=llm, ) From 1386ce97741551774c2611b923e4722762ebb39b Mon Sep 17 00:00:00 2001 From: Oshoma Momoh Date: Thu, 16 Nov 2023 11:39:33 -0500 Subject: [PATCH 18/18] Rename `do_reflect` option to `do-reflect` We should use dashes instead of underscores in command line options. This confirms with the [Gnu style guide](https://www.gnu.org/software/libc/manual/html_node/Argument-Syntax.html) --- src/sherpa_ai/config/task_config.py | 2 +- src/tests/unit_tests/config/test_task_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sherpa_ai/config/task_config.py b/src/sherpa_ai/config/task_config.py index 54408e60..f3a89c70 100644 --- a/src/sherpa_ai/config/task_config.py +++ b/src/sherpa_ai/config/task_config.py @@ -41,7 +41,7 @@ def from_config(cls, configs: List[str]) -> "AgentConfig": help="site to be used for the Google search tool.", ) parser.add_argument( - "--do_reflect", + "--do-reflect", action="store_true", help="enable performing the reflection step for each agent.", ) diff --git a/src/tests/unit_tests/config/test_task_config.py b/src/tests/unit_tests/config/test_task_config.py index 0d49f0e0..22fbc559 100644 --- a/src/tests/unit_tests/config/test_task_config.py +++ b/src/tests/unit_tests/config/test_task_config.py @@ -5,7 +5,7 @@ def test_all_parameters_parse_successfully(): site = "https://www.google.com" - input_str = f"Test input. --verbose --gsite {site} --do_reflect" + input_str = f"Test input. --verbose --gsite {site} --do-reflect" parsed, config = AgentConfig.from_input(input_str)