From e794e6bfa37efd20ae504caba5bc6676f91b07f5 Mon Sep 17 00:00:00 2001 From: IANTHEREAL Date: Sat, 9 Dec 2023 10:57:58 +0800 Subject: [PATCH 1/8] Filter out candidates with the same name but different instructions, file IDs, and function names --- .../agentchat/contrib/gpt_assistant_agent.py | 62 ++++++- autogen/oai/openai_utils.py | 2 +- test/agentchat/contrib/test_gpt_assistant.py | 152 +++++++++++++++++- 3 files changed, 207 insertions(+), 9 deletions(-) diff --git a/autogen/agentchat/contrib/gpt_assistant_agent.py b/autogen/agentchat/contrib/gpt_assistant_agent.py index 2fd431e99a22..c0242451245a 100644 --- a/autogen/agentchat/contrib/gpt_assistant_agent.py +++ b/autogen/agentchat/contrib/gpt_assistant_agent.py @@ -59,9 +59,14 @@ def __init__( if openai_assistant_id is None: # try to find assistant by name first candidate_assistants = retrieve_assistants_by_name(self._openai_client, name) + if len(candidate_assistants) > 0: + # Filter out candidates with the same name but different instructions, file IDs, and function names. + candidate_assistants = self.find_matching_assistant( + candidate_assistants, instructions, llm_config.get("tools", []), llm_config.get("file_ids", []) + ) if len(candidate_assistants) == 0: - logger.warning(f"assistant {name} does not exist, creating a new assistant") + logger.warning("assistant %s does not exist, creating a new assistant", name) # create a new assistant if instructions is None: logger.warning( @@ -76,11 +81,11 @@ def __init__( file_ids=llm_config.get("file_ids", []), ) else: - if len(candidate_assistants) > 1: - logger.warning( - f"Multiple assistants with name {name} found. Using the first assistant in the list. " - f"Please specify the assistant ID in llm_config to use a specific assistant." - ) + logger.warning( + "assistant %s already exists, using the first matching assistant: %s", + name, + candidate_assistants[0].__dict__, + ) self._openai_assistant = candidate_assistants[0] else: # retrieve an existing assistant @@ -368,3 +373,48 @@ def delete_assistant(self): """Delete the assistant from OAI assistant API""" logger.warning("Permanently deleting assistant...") self._openai_client.beta.assistants.delete(self.assistant_id) + + def find_matching_assistant(self, candidate_assistants, instructions, tools, file_ids): + """ + Find the matching assistant from a list of candidate assistants. + Filter out candidates with the same name but different instructions, file IDs, and function names. + TODO: implement accurate match based on assistant metadata fields. + """ + matching_assistants = [] + + # Preprocess the required tools for faster comparison + required_tool_types = set(tool.get("type") for tool in tools) + required_function_names = set( + tool.get("function", {}).get("name") + for tool in tools + if tool.get("type") not in ["code_interpreter", "retrieval"] + ) + required_file_ids = set(file_ids) # Convert file_ids to a set for unordered comparison + + for assistant in candidate_assistants: + # Check if instructions are similar + if instructions and instructions != getattr(assistant, "instructions", None): + print("instructions not match") + print(instructions, getattr(assistant, "instructions", None)) + continue + + # Preprocess the assistant's tools + assistant_tool_types = set(tool.type for tool in assistant.tools) + assistant_function_names = set(tool.function.name for tool in assistant.tools if hasattr(tool, "function")) + assistant_file_ids = set(getattr(assistant, "file_ids", [])) # Convert to set for comparison + + # Check if the tool types, function names, and file IDs match + if required_tool_types != assistant_tool_types or required_function_names != assistant_function_names: + print("tools not match") + print(required_tool_types, assistant_tool_types) + print(required_function_names, assistant_function_names) + continue + if required_file_ids != assistant_file_ids: + print("file_ids not match") + print(required_file_ids, assistant_file_ids) + continue + + # Append assistant to matching list if all conditions are met + matching_assistants.append(assistant) + + return matching_assistants diff --git a/autogen/oai/openai_utils.py b/autogen/oai/openai_utils.py index 966646cdb802..9c02b1e3aa13 100644 --- a/autogen/oai/openai_utils.py +++ b/autogen/oai/openai_utils.py @@ -413,7 +413,7 @@ def config_list_from_dotenv( return config_list -def retrieve_assistants_by_name(client, name) -> str: +def retrieve_assistants_by_name(client, name): """ Return the assistants with the given name from OAI assistant API """ diff --git a/test/agentchat/contrib/test_gpt_assistant.py b/test/agentchat/contrib/test_gpt_assistant.py index fae85bbf384d..307e977b53f6 100644 --- a/test/agentchat/contrib/test_gpt_assistant.py +++ b/test/agentchat/contrib/test_gpt_assistant.py @@ -222,17 +222,46 @@ def test_assistant_retrieval(): name = "For test_assistant_retrieval" + function_1_schema = { + "name": "call_function_1", + "parameters": {"type": "object", "properties": {}, "required": []}, + "description": "This is a test function 1", + } + function_2_schema = { + "name": "call_function_1", + "parameters": {"type": "object", "properties": {}, "required": []}, + "description": "This is a test function 2", + } + + openai_client = OpenAIWrapper(config_list=config_list)._clients[0] + current_file_path = os.path.abspath(__file__) + file_1 = openai_client.files.create(file=open(current_file_path, "rb"), purpose="assistants") + file_2 = openai_client.files.create(file=open(current_file_path, "rb"), purpose="assistants") + + all_llm_config = { + "tools": [ + {"type": "function", "function": function_1_schema}, + {"type": "function", "function": function_2_schema}, + {"type": "retrieval"}, + {"type": "code_interpreter"}, + ], + "file_ids": [file_1.id, file_2.id], + "config_list": config_list, + } + + name = "For test_gpt_assistant_chat" + assistant_first = GPTAssistantAgent( name, instructions="This is a test", - llm_config={"config_list": config_list}, + llm_config=all_llm_config, ) candidate_first = retrieve_assistants_by_name(assistant_first.openai_client, name) assistant_second = GPTAssistantAgent( name, instructions="This is a test", - llm_config={"config_list": config_list}, + llm_config=all_llm_config, ) candidate_second = retrieve_assistants_by_name(assistant_second.openai_client, name) @@ -243,7 +272,125 @@ def test_assistant_retrieval(): # Not found error is expected because the same assistant can not be deleted twice pass + openai_client.files.delete(file_1.id) + openai_client.files.delete(file_2.id) + assert candidate_first == candidate_second + assert len(candidate_first) == 1 + + candidates = retrieve_assistants_by_name(openai_client, name) + assert len(candidates) == 0 + + +@pytest.mark.skipif( + sys.platform in ["darwin", "win32"] or skip_test, + reason="do not run on MacOS or windows or dependency is not installed", +) +def test_assistant_mismatch_retrieval(): + """Test function to check if the GPTAssistantAgent can filter out the mismatch assistant""" + + name = "For test_assistant_retrieval" + + function_1_schema = { + "name": "call_function", + "parameters": {"type": "object", "properties": {}, "required": []}, + "description": "This is a test function 1", + } + function_2_schema = { + "name": "call_function", + "parameters": {"type": "object", "properties": {}, "required": []}, + "description": "This is a test function 2", + } + function_3_schema = { + "name": "call_function_other", + "parameters": {"type": "object", "properties": {}, "required": []}, + "description": "This is a test function 3", + } + + openai_client = OpenAIWrapper(config_list=config_list)._clients[0] + current_file_path = os.path.abspath(__file__) + file_1 = openai_client.files.create(file=open(current_file_path, "rb"), purpose="assistants") + file_2 = openai_client.files.create(file=open(current_file_path, "rb"), purpose="assistants") + + all_llm_config = { + "tools": [ + {"type": "function", "function": function_1_schema}, + {"type": "function", "function": function_2_schema}, + {"type": "retrieval"}, + {"type": "code_interpreter"}, + ], + "file_ids": [file_1.id, file_2.id], + "config_list": config_list, + } + + name = "For test_gpt_assistant_chat" + + assistant_first = GPTAssistantAgent( + name, + instructions="This is a test", + llm_config=all_llm_config, + ) + candidate_first = retrieve_assistants_by_name(assistant_first.openai_client, name) + assert len(candidate_first) == 1 + + # test instructions mismatch + assistant_instructions_mistaching = GPTAssistantAgent( + name, + instructions="This is a test for mismatch instructions", + llm_config=all_llm_config, + ) + candidate_instructions_mistaching = retrieve_assistants_by_name( + assistant_instructions_mistaching.openai_client, name + ) + assert len(candidate_instructions_mistaching) == 2 + + # test mismatch fild ids + file_ids_mismatch_llm_config = { + "tools": [ + {"type": "code_interpreter"}, + {"type": "retrieval"}, + {"type": "function", "function": function_2_schema}, + {"type": "function", "function": function_1_schema}, + ], + "file_ids": [file_2.id], + "config_list": config_list, + } + assistant_file_ids_mismatch = GPTAssistantAgent( + name, + instructions="This is a test", + llm_config=file_ids_mismatch_llm_config, + ) + candidate_file_ids_mismatch = retrieve_assistants_by_name(assistant_file_ids_mismatch.openai_client, name) + assert len(candidate_file_ids_mismatch) == 3 + + # test tools mismatch + tools_mismatch_llm_config = { + "tools": [ + {"type": "code_interpreter"}, + {"type": "retrieval"}, + {"type": "function", "function": function_3_schema}, + ], + "file_ids": [file_2.id, file_1.id], + "config_list": config_list, + } + assistant_tools_mistaching = GPTAssistantAgent( + name, + instructions="This is a test", + llm_config=tools_mismatch_llm_config, + ) + candidate_tools_mismatch = retrieve_assistants_by_name(assistant_tools_mistaching.openai_client, name) + assert len(candidate_tools_mismatch) == 4 + + openai_client.files.delete(file_1.id) + openai_client.files.delete(file_2.id) + + assistant_first.delete_assistant() + assistant_instructions_mistaching.delete_assistant() + assistant_file_ids_mismatch.delete_assistant() + assistant_tools_mistaching.delete_assistant() + + candidates = retrieve_assistants_by_name(openai_client, name) + assert len(candidates) == 0 if __name__ == "__main__": @@ -252,3 +399,4 @@ def test_assistant_retrieval(): test_gpt_assistant_instructions_overwrite() test_gpt_assistant_existing_no_instructions() test_get_assistant_files() + test_assistant_mismatch_retrieval() From f2102fb2ee43ab96187cea5ab6ff3bb7e209290c Mon Sep 17 00:00:00 2001 From: IANTHEREAL Date: Sun, 10 Dec 2023 11:42:26 +0800 Subject: [PATCH 2/8] polish --- autogen/agentchat/contrib/gpt_assistant_agent.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/autogen/agentchat/contrib/gpt_assistant_agent.py b/autogen/agentchat/contrib/gpt_assistant_agent.py index c0242451245a..350196c4c8eb 100644 --- a/autogen/agentchat/contrib/gpt_assistant_agent.py +++ b/autogen/agentchat/contrib/gpt_assistant_agent.py @@ -394,8 +394,7 @@ def find_matching_assistant(self, candidate_assistants, instructions, tools, fil for assistant in candidate_assistants: # Check if instructions are similar if instructions and instructions != getattr(assistant, "instructions", None): - print("instructions not match") - print(instructions, getattr(assistant, "instructions", None)) + logger.warning("instructions not match, skip assistant: %s", getattr(assistant, "instructions", None)) continue # Preprocess the assistant's tools @@ -405,13 +404,14 @@ def find_matching_assistant(self, candidate_assistants, instructions, tools, fil # Check if the tool types, function names, and file IDs match if required_tool_types != assistant_tool_types or required_function_names != assistant_function_names: - print("tools not match") - print(required_tool_types, assistant_tool_types) - print(required_function_names, assistant_function_names) + logger.warning( + "tools not match, skip assistant: tools %s, functions %s", + assistant_tool_types, + assistant_function_names, + ) continue if required_file_ids != assistant_file_ids: - print("file_ids not match") - print(required_file_ids, assistant_file_ids) + logger.warning("file_ids not match, skip assistant: %s", assistant_file_ids) continue # Append assistant to matching list if all conditions are met From 87c81896a0023d45143f26fd6ab84802750ec4b8 Mon Sep 17 00:00:00 2001 From: IANTHEREAL Date: Wed, 13 Dec 2023 15:37:21 +0800 Subject: [PATCH 3/8] improve log --- autogen/agentchat/contrib/gpt_assistant_agent.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/autogen/agentchat/contrib/gpt_assistant_agent.py b/autogen/agentchat/contrib/gpt_assistant_agent.py index 350196c4c8eb..f32a7ed3397b 100644 --- a/autogen/agentchat/contrib/gpt_assistant_agent.py +++ b/autogen/agentchat/contrib/gpt_assistant_agent.py @@ -13,7 +13,6 @@ logger = logging.getLogger(__name__) - class GPTAssistantAgent(ConversableAgent): """ An experimental AutoGen agent class that leverages the OpenAI Assistant API for conversational capabilities. @@ -394,7 +393,11 @@ def find_matching_assistant(self, candidate_assistants, instructions, tools, fil for assistant in candidate_assistants: # Check if instructions are similar if instructions and instructions != getattr(assistant, "instructions", None): - logger.warning("instructions not match, skip assistant: %s", getattr(assistant, "instructions", None)) + logger.warning( + "instructions not match, skip assistant(%s): %s", + assistant.id, + getattr(assistant, "instructions", None), + ) continue # Preprocess the assistant's tools @@ -405,13 +408,14 @@ def find_matching_assistant(self, candidate_assistants, instructions, tools, fil # Check if the tool types, function names, and file IDs match if required_tool_types != assistant_tool_types or required_function_names != assistant_function_names: logger.warning( - "tools not match, skip assistant: tools %s, functions %s", + "tools not match, skip assistant(%s): tools %s, functions %s", + assistant.id, assistant_tool_types, assistant_function_names, ) continue if required_file_ids != assistant_file_ids: - logger.warning("file_ids not match, skip assistant: %s", assistant_file_ids) + logger.warning("file_ids not match, skip assistant(%s): %s", assistant.id, assistant_file_ids) continue # Append assistant to matching list if all conditions are met From f84ec741f3e2ff8895a95264ce647c0252246112 Mon Sep 17 00:00:00 2001 From: IANTHEREAL Date: Wed, 13 Dec 2023 16:08:03 +0800 Subject: [PATCH 4/8] improving log --- autogen/agentchat/contrib/gpt_assistant_agent.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/autogen/agentchat/contrib/gpt_assistant_agent.py b/autogen/agentchat/contrib/gpt_assistant_agent.py index f32a7ed3397b..71b143afed46 100644 --- a/autogen/agentchat/contrib/gpt_assistant_agent.py +++ b/autogen/agentchat/contrib/gpt_assistant_agent.py @@ -13,6 +13,7 @@ logger = logging.getLogger(__name__) + class GPTAssistantAgent(ConversableAgent): """ An experimental AutoGen agent class that leverages the OpenAI Assistant API for conversational capabilities. @@ -65,7 +66,7 @@ def __init__( ) if len(candidate_assistants) == 0: - logger.warning("assistant %s does not exist, creating a new assistant", name) + logger.warning("Assistant %s does not exist, creating a new assistant", name) # create a new assistant if instructions is None: logger.warning( @@ -81,7 +82,7 @@ def __init__( ) else: logger.warning( - "assistant %s already exists, using the first matching assistant: %s", + "Assistant %s already exists, using the first matching assistant: %s", name, candidate_assistants[0].__dict__, ) From 2943d3322659e5cc0d123381820cf2372d354e6f Mon Sep 17 00:00:00 2001 From: IANTHEREAL Date: Wed, 13 Dec 2023 16:21:00 +0800 Subject: [PATCH 5/8] improve log --- autogen/agentchat/contrib/gpt_assistant_agent.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/autogen/agentchat/contrib/gpt_assistant_agent.py b/autogen/agentchat/contrib/gpt_assistant_agent.py index 71b143afed46..00559f521603 100644 --- a/autogen/agentchat/contrib/gpt_assistant_agent.py +++ b/autogen/agentchat/contrib/gpt_assistant_agent.py @@ -45,7 +45,7 @@ def __init__( - tools: Give Assistants access to OpenAI-hosted tools like Code Interpreter and Knowledge Retrieval, or build your own tools using Function calling. ref https://platform.openai.com/docs/assistants/tools - file_ids: files used by retrieval in run - overwrite_instructions (bool): whether to overwrite the instructions of an existing assistant. + overwrite_instructions (bool): whether to overwrite the instructions of an existing assistant. This parameter is in effect only when assistant_id is specified in llm_config. kwargs (dict): Additional configuration options for the agent. - verbose (bool): If set to True, enables more detailed output from the assistant thread. - Other kwargs: Except verbose, others are passed directly to ConversableAgent. @@ -66,7 +66,7 @@ def __init__( ) if len(candidate_assistants) == 0: - logger.warning("Assistant %s does not exist, creating a new assistant", name) + logger.warning("No matching assistant found, creating a new assistant") # create a new assistant if instructions is None: logger.warning( @@ -82,8 +82,7 @@ def __init__( ) else: logger.warning( - "Assistant %s already exists, using the first matching assistant: %s", - name, + "Matching assistant found, using the first matching assistant: %s", candidate_assistants[0].__dict__, ) self._openai_assistant = candidate_assistants[0] From a30111d38dd87229dfa5ddd346c77f4feb779d77 Mon Sep 17 00:00:00 2001 From: gagb Date: Sun, 17 Dec 2023 01:32:08 -0600 Subject: [PATCH 6/8] Improve function signature (#2) --- autogen/oai/openai_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/autogen/oai/openai_utils.py b/autogen/oai/openai_utils.py index 9c02b1e3aa13..e078542d098b 100644 --- a/autogen/oai/openai_utils.py +++ b/autogen/oai/openai_utils.py @@ -6,6 +6,8 @@ import logging from dotenv import find_dotenv, load_dotenv +from openai import OpenAI +from openai.types.beta.assistant import Assistant NON_CACHE_KEY = ["api_key", "base_url", "api_type", "api_version"] @@ -413,7 +415,7 @@ def config_list_from_dotenv( return config_list -def retrieve_assistants_by_name(client, name): +def retrieve_assistants_by_name(client: OpenAI, name: str) -> List[Assistant]: """ Return the assistants with the given name from OAI assistant API """ From ab12725b5f5e8f8abf5b876a962a6977dd8802b4 Mon Sep 17 00:00:00 2001 From: IANTHEREAL Date: Fri, 22 Dec 2023 19:54:49 +0800 Subject: [PATCH 7/8] try to fix ci --- autogen/oai/openai_utils.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/autogen/oai/openai_utils.py b/autogen/oai/openai_utils.py index e078542d098b..6c4ebf671b58 100644 --- a/autogen/oai/openai_utils.py +++ b/autogen/oai/openai_utils.py @@ -6,8 +6,14 @@ import logging from dotenv import find_dotenv, load_dotenv -from openai import OpenAI -from openai.types.beta.assistant import Assistant +try: + from openai import OpenAI + from openai.types.beta.assistant import Assistant + + ERROR = None +except ImportError: + ERROR = ImportError("Please install openai>=1 to use autogen.OpenAIWrapper.") + OpenAI = object NON_CACHE_KEY = ["api_key", "base_url", "api_type", "api_version"] @@ -419,6 +425,8 @@ def retrieve_assistants_by_name(client: OpenAI, name: str) -> List[Assistant]: """ Return the assistants with the given name from OAI assistant API """ + if ERROR: + raise ERROR assistants = client.beta.assistants.list() candidate_assistants = [] for assistant in assistants.data: From 8739e8a2f8e0d00621737b7b969505fee155a896 Mon Sep 17 00:00:00 2001 From: IANTHEREAL Date: Fri, 22 Dec 2023 19:57:31 +0800 Subject: [PATCH 8/8] try to fix ci --- autogen/oai/openai_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/autogen/oai/openai_utils.py b/autogen/oai/openai_utils.py index 6c4ebf671b58..a2ddfb8535c6 100644 --- a/autogen/oai/openai_utils.py +++ b/autogen/oai/openai_utils.py @@ -14,6 +14,7 @@ except ImportError: ERROR = ImportError("Please install openai>=1 to use autogen.OpenAIWrapper.") OpenAI = object + Assistant = object NON_CACHE_KEY = ["api_key", "base_url", "api_type", "api_version"]