diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py index 5e1e7f3d2038..4c94b497d8a0 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py @@ -45,6 +45,7 @@ HandoffMessage, MemoryQueryEvent, ModelClientStreamingChunkEvent, + StructuredMessage, TextMessage, ThoughtEvent, ToolCallExecutionEvent, @@ -102,12 +103,25 @@ class AssistantAgent(BaseChatAgent, Component[AssistantAgentConfig]): .. image:: ../../images/assistant-agent.svg - Tool call behavior: + **Structured output:** - * If the model returns no tool call, then the response is immediately returned as a :class:`~autogen_agentchat.messages.TextMessage` in :attr:`~autogen_agentchat.base.Response.chat_message`. + If the `output_content_type` is set, the agent will respond with a :class:`~autogen_agentchat.messages.StructuredMessage` + instead of a :class:`~autogen_agentchat.messages.TextMessage` in the final response by default. + + .. note:: + + Currently, setting `output_content_type` prevents the agent from being + able to call `load_component` and `dum_component` methods for serializable + configuration. This will be fixed soon in the future. + + **Tool call behavior:** + + * If the model returns no tool call, then the response is immediately returned as a :class:`~autogen_agentchat.messages.TextMessage` or a :class:`~autogen_agentchat.messages.StructuredMessage` (when using structured output) in :attr:`~autogen_agentchat.base.Response.chat_message`. * When the model returns tool calls, they will be executed right away: - - When `reflect_on_tool_use` is False (default), the tool call results are returned as a :class:`~autogen_agentchat.messages.ToolCallSummaryMessage` in :attr:`~autogen_agentchat.base.Response.chat_message`. `tool_call_summary_format` can be used to customize the tool call summary. - - When `reflect_on_tool_use` is True, the another model inference is made using the tool calls and results, and the text response is returned as a :class:`~autogen_agentchat.messages.TextMessage` in :attr:`~autogen_agentchat.base.Response.chat_message`. + - When `reflect_on_tool_use` is False, the tool call results are returned as a :class:`~autogen_agentchat.messages.ToolCallSummaryMessage` in :attr:`~autogen_agentchat.base.Response.chat_message`. `tool_call_summary_format` can be used to customize the tool call summary. + - When `reflect_on_tool_use` is True, the another model inference is made using the tool calls and results, and final response is returned as a :class:`~autogen_agentchat.messages.TextMessage` or a :class:`~autogen_agentchat.messages.StructuredMessage` (when using structured output) in :attr:`~autogen_agentchat.base.Response.chat_message`. + - `reflect_on_tool_use` is set to `True` by default when `output_content_type` is set. + - `reflect_on_tool_use` is set to `False` by default when `output_content_type` is not set. * If the model returns multiple tool calls, they will be executed concurrently. To disable parallel tool calls you need to configure the model client. For example, set `parallel_tool_calls=False` for :class:`~autogen_ext.models.openai.OpenAIChatCompletionClient` and :class:`~autogen_ext.models.openai.AzureOpenAIChatCompletionClient`. .. tip:: @@ -116,7 +130,7 @@ class AssistantAgent(BaseChatAgent, Component[AssistantAgentConfig]): especially if another agent is expecting them in a specific format. Use `tool_call_summary_format` to customize the tool call summary, if needed. - Hand off behavior: + **Hand off behavior:** * If a handoff is triggered, a :class:`~autogen_agentchat.messages.HandoffMessage` will be returned in :attr:`~autogen_agentchat.base.Response.chat_message`. * If there are tool calls, they will also be executed right away before returning the handoff. @@ -128,16 +142,18 @@ class AssistantAgent(BaseChatAgent, Component[AssistantAgentConfig]): To avoid this, disable parallel tool calls in the model client configuration. - Limit context size sent to the model: + **Limit context size sent to the model:** You can limit the number of messages sent to the model by setting the `model_context` parameter to a :class:`~autogen_core.model_context.BufferedChatCompletionContext`. This will limit the number of recent messages sent to the model and can be useful when the model has a limit on the number of tokens it can process. + Another option is to use a :class:`~autogen_core.model_context.TokenLimitedChatCompletionContext` + which will limit the number of tokens sent to the model. You can also create your own model context by subclassing :class:`~autogen_core.model_context.ChatCompletionContext`. - Streaming mode: + **Streaming mode:** The assistant agent can be used in streaming mode by setting `model_client_stream=True`. In this mode, the :meth:`on_messages_stream` and :meth:`BaseChatAgent.run_stream` methods will also yield @@ -161,8 +177,14 @@ class AssistantAgent(BaseChatAgent, Component[AssistantAgentConfig]): :meth:`on_messages_stream` and :meth:`BaseChatAgent.run_stream` methods will also yield :class:`~autogen_agentchat.messages.ModelClientStreamingChunkEvent` messages as the model client produces chunks of response. Defaults to `False`. reflect_on_tool_use (bool, optional): If `True`, the agent will make another model inference using the tool call and result - to generate a response. If `False`, the tool call result will be returned as the response. Defaults to `False`. - tool_call_summary_format (str, optional): The format string used to create a tool call summary for every tool call result. + to generate a response. If `False`, the tool call result will be returned as the response. By default, if `output_content_type` is set, this will be `True`; + if `output_content_type` is not set, this will be `False`. + output_content_type (type[BaseModel] | None, optional): The output content type for :class:`~autogen_agentchat.messages.StructuredMessage` response as a Pydantic model. + This will be used with the model client to generate structured output. + If this is set, the agent will respond with a :class:`~autogen_agentchat.messages.StructuredMessage` instead of a :class:`~autogen_agentchat.messages.TextMessage` + in the final response, unless `reflect_on_tool_use` is `False` and a tool call is made. + tool_call_summary_format (str, optional): The format string used to create the content for a :class:`~autogen_agentchat.messages.ToolCallSummaryMessage` response. + The format string is used to format the tool call summary for every tool call result. Defaults to "{result}". When `reflect_on_tool_use` is `False`, a concatenation of all the tool call summaries, separated by a new line character ('\\n') will be returned as the response. @@ -348,10 +370,9 @@ def sentiment_analysis(text: str) -> str: # which is required for structured output mode. tool = FunctionTool(sentiment_analysis, description="Sentiment Analysis", strict=True) - # Create an OpenAIChatCompletionClient instance that uses the structured output format. + # Create an OpenAIChatCompletionClient instance that supports structured output. model_client = OpenAIChatCompletionClient( model="gpt-4o-mini", - response_format=AgentResponse, # type: ignore ) # Create an AssistantAgent instance that uses the tool and model client. @@ -360,7 +381,7 @@ def sentiment_analysis(text: str) -> str: model_client=model_client, tools=[tool], system_message="Use the tool to analyze sentiment.", - reflect_on_tool_use=True, # Use reflection to have the agent generate a formatted response. + output_content_type=AgentResponse, ) @@ -611,25 +632,17 @@ def __init__( str | None ) = "You are a helpful AI assistant. Solve tasks using your tools. Reply with TERMINATE when the task has been completed.", model_client_stream: bool = False, - reflect_on_tool_use: bool = False, + reflect_on_tool_use: bool | None = None, tool_call_summary_format: str = "{result}", + output_content_type: type[BaseModel] | None = None, memory: Sequence[Memory] | None = None, metadata: Dict[str, str] | None = None, ): super().__init__(name=name, description=description) self._metadata = metadata or {} - if reflect_on_tool_use and ModelFamily.is_claude(model_client.model_info["family"]): - warnings.warn( - "Claude models may not work with reflection on tool use because Claude requires that any requests including a previous tool use or tool result must include the original tools definition." - "Consider setting reflect_on_tool_use to False. " - "As an alternative, consider calling the agent in a loop until it stops producing tool calls. " - "See [Single-Agent Team](https://microsoft.github.io/autogen/stable/user-guide/agentchat-user-guide/tutorial/teams.html#single-agent-team) " - "for more details.", - UserWarning, - stacklevel=2, - ) self._model_client = model_client self._model_client_stream = model_client_stream + self._output_content_type: type[BaseModel] | None = output_content_type self._memory = None if memory is not None: if isinstance(memory, list): @@ -692,17 +705,37 @@ def __init__( else: self._model_context = UnboundedChatCompletionContext() - self._reflect_on_tool_use = reflect_on_tool_use + if self._output_content_type is not None and reflect_on_tool_use is None: + # If output_content_type is set, we need to reflect on tool use by default. + self._reflect_on_tool_use = True + elif reflect_on_tool_use is None: + self._reflect_on_tool_use = False + else: + self._reflect_on_tool_use = reflect_on_tool_use + if self._reflect_on_tool_use and ModelFamily.is_claude(model_client.model_info["family"]): + warnings.warn( + "Claude models may not work with reflection on tool use because Claude requires that any requests including a previous tool use or tool result must include the original tools definition." + "Consider setting reflect_on_tool_use to False. " + "As an alternative, consider calling the agent in a loop until it stops producing tool calls. " + "See [Single-Agent Team](https://microsoft.github.io/autogen/stable/user-guide/agentchat-user-guide/tutorial/teams.html#single-agent-team) " + "for more details.", + UserWarning, + stacklevel=2, + ) self._tool_call_summary_format = tool_call_summary_format self._is_running = False @property def produced_message_types(self) -> Sequence[type[BaseChatMessage]]: - message_types: List[type[BaseChatMessage]] = [TextMessage] + message_types: List[type[BaseChatMessage]] = [] if self._handoffs: message_types.append(HandoffMessage) if self._tools: message_types.append(ToolCallSummaryMessage) + if self._output_content_type: + message_types.append(StructuredMessage[self._output_content_type]) # type: ignore[name-defined] + else: + message_types.append(TextMessage) return tuple(message_types) @property @@ -737,6 +770,7 @@ async def on_messages_stream( model_client_stream = self._model_client_stream reflect_on_tool_use = self._reflect_on_tool_use tool_call_summary_format = self._tool_call_summary_format + output_content_type = self._output_content_type # STEP 1: Add new user/handoff messages to the model context await self._add_messages_to_context( @@ -765,6 +799,7 @@ async def on_messages_stream( handoff_tools=handoff_tools, agent_name=agent_name, cancellation_token=cancellation_token, + output_content_type=output_content_type, ): if isinstance(inference_output, CreateResult): model_result = inference_output @@ -804,6 +839,7 @@ async def on_messages_stream( model_client_stream=model_client_stream, reflect_on_tool_use=reflect_on_tool_use, tool_call_summary_format=tool_call_summary_format, + output_content_type=output_content_type, ): yield output_event @@ -853,6 +889,7 @@ async def _call_llm( handoff_tools: List[BaseTool[Any, Any]], agent_name: str, cancellation_token: CancellationToken, + output_content_type: type[BaseModel] | None, ) -> AsyncGenerator[Union[CreateResult, ModelClientStreamingChunkEvent], None]: """ Perform a model inference and yield either streaming chunk events or the final CreateResult. @@ -865,7 +902,10 @@ async def _call_llm( if model_client_stream: model_result: Optional[CreateResult] = None async for chunk in model_client.create_stream( - llm_messages, tools=all_tools, cancellation_token=cancellation_token + llm_messages, + tools=all_tools, + json_output=output_content_type, + cancellation_token=cancellation_token, ): if isinstance(chunk, CreateResult): model_result = chunk @@ -878,7 +918,10 @@ async def _call_llm( yield model_result else: model_result = await model_client.create( - llm_messages, tools=all_tools, cancellation_token=cancellation_token + llm_messages, + tools=all_tools, + cancellation_token=cancellation_token, + json_output=output_content_type, ) yield model_result @@ -898,6 +941,7 @@ async def _process_model_result( model_client_stream: bool, reflect_on_tool_use: bool, tool_call_summary_format: str, + output_content_type: type[BaseModel] | None, ) -> AsyncGenerator[BaseAgentEvent | BaseChatMessage | Response, None]: """ Handle final or partial responses from model_result, including tool calls, handoffs, @@ -906,14 +950,25 @@ async def _process_model_result( # If direct text response (string) if isinstance(model_result.content, str): - yield Response( - chat_message=TextMessage( - content=model_result.content, - source=agent_name, - models_usage=model_result.usage, - ), - inner_messages=inner_messages, - ) + if output_content_type: + content = output_content_type.model_validate_json(model_result.content) + yield Response( + chat_message=StructuredMessage[output_content_type]( # type: ignore[valid-type] + content=content, + source=agent_name, + models_usage=model_result.usage, + ), + inner_messages=inner_messages, + ) + else: + yield Response( + chat_message=TextMessage( + content=model_result.content, + source=agent_name, + models_usage=model_result.usage, + ), + inner_messages=inner_messages, + ) return # Otherwise, we have function calls @@ -977,6 +1032,7 @@ async def _process_model_result( model_context=model_context, agent_name=agent_name, inner_messages=inner_messages, + output_content_type=output_content_type, ): yield reflection_response else: @@ -1062,6 +1118,7 @@ async def _reflect_on_tool_use_flow( model_context: ChatCompletionContext, agent_name: str, inner_messages: List[BaseAgentEvent | BaseChatMessage], + output_content_type: type[BaseModel] | None, ) -> AsyncGenerator[Response | ModelClientStreamingChunkEvent | ThoughtEvent, None]: """ If reflect_on_tool_use=True, we do another inference based on tool results @@ -1073,7 +1130,10 @@ async def _reflect_on_tool_use_flow( reflection_result: Optional[CreateResult] = None if model_client_stream: - async for chunk in model_client.create_stream(llm_messages): + async for chunk in model_client.create_stream( + llm_messages, + json_output=output_content_type, + ): if isinstance(chunk, CreateResult): reflection_result = chunk elif isinstance(chunk, str): @@ -1081,7 +1141,7 @@ async def _reflect_on_tool_use_flow( else: raise RuntimeError(f"Invalid chunk type: {type(chunk)}") else: - reflection_result = await model_client.create(llm_messages) + reflection_result = await model_client.create(llm_messages, json_output=output_content_type) if not reflection_result or not isinstance(reflection_result.content, str): raise RuntimeError("Reflect on tool use produced no valid text response.") @@ -1101,14 +1161,25 @@ async def _reflect_on_tool_use_flow( ) ) - yield Response( - chat_message=TextMessage( - content=reflection_result.content, - source=agent_name, - models_usage=reflection_result.usage, - ), - inner_messages=inner_messages, - ) + if output_content_type: + content = output_content_type.model_validate_json(reflection_result.content) + yield Response( + chat_message=StructuredMessage[output_content_type]( # type: ignore[valid-type] + content=content, + source=agent_name, + models_usage=reflection_result.usage, + ), + inner_messages=inner_messages, + ) + else: + yield Response( + chat_message=TextMessage( + content=reflection_result.content, + source=agent_name, + models_usage=reflection_result.usage, + ), + inner_messages=inner_messages, + ) @staticmethod def _summarize_tool_use( @@ -1206,6 +1277,9 @@ def _get_compatible_context(model_client: ChatCompletionClient, messages: List[L def _to_config(self) -> AssistantAgentConfig: """Convert the assistant agent to a declarative config.""" + if self._output_content_type: + raise ValueError("AssistantAgent with output_content_type does not support declarative config.") + return AssistantAgentConfig( name=self.name, model_client=self._model_client.dump_component(), @@ -1226,6 +1300,7 @@ def _to_config(self) -> AssistantAgentConfig: @classmethod def _from_config(cls, config: AssistantAgentConfig) -> Self: """Create an assistant agent from a declarative config.""" + return cls( name=config.name, model_client=ChatCompletionClient.load_component(config.model_client), diff --git a/python/packages/autogen-agentchat/tests/test_assistant_agent.py b/python/packages/autogen-agentchat/tests/test_assistant_agent.py index 8de0665c91d2..38158391909c 100644 --- a/python/packages/autogen-agentchat/tests/test_assistant_agent.py +++ b/python/packages/autogen-agentchat/tests/test_assistant_agent.py @@ -401,6 +401,147 @@ async def test_run_with_parallel_tools_with_empty_call_ids() -> None: assert state == state2 +@pytest.mark.asyncio +async def test_output_format() -> None: + class AgentResponse(BaseModel): + response: str + status: str + + model_client = ReplayChatCompletionClient( + [ + CreateResult( + finish_reason="stop", + content=AgentResponse(response="Hello", status="success").model_dump_json(), + usage=RequestUsage(prompt_tokens=10, completion_tokens=5), + cached=False, + ), + ] + ) + agent = AssistantAgent( + "test_agent", + model_client=model_client, + output_content_type=AgentResponse, + ) + assert StructuredMessage[AgentResponse] in agent.produced_message_types + assert TextMessage not in agent.produced_message_types + + result = await agent.run() + assert len(result.messages) == 1 + assert isinstance(result.messages[0], StructuredMessage) + assert isinstance(result.messages[0].content, AgentResponse) # type: ignore[reportUnknownMemberType] + assert result.messages[0].content.response == "Hello" + assert result.messages[0].content.status == "success" + + # Test streaming. + agent = AssistantAgent( + "test_agent", + model_client=model_client, + model_client_stream=True, + output_content_type=AgentResponse, + ) + model_client.reset() + stream = agent.run_stream() + stream_result: TaskResult | None = None + async for message in stream: + if isinstance(message, TaskResult): + stream_result = message + assert stream_result is not None + assert len(stream_result.messages) == 1 + assert isinstance(stream_result.messages[0], StructuredMessage) + assert isinstance(stream_result.messages[0].content, AgentResponse) # type: ignore[reportUnknownMemberType] + assert stream_result.messages[0].content.response == "Hello" + assert stream_result.messages[0].content.status == "success" + + +@pytest.mark.asyncio +async def test_reflection_output_format() -> None: + class AgentResponse(BaseModel): + response: str + status: str + + model_client = ReplayChatCompletionClient( + [ + CreateResult( + finish_reason="function_calls", + content=[FunctionCall(id="1", arguments=json.dumps({"input": "task"}), name="_pass_function")], + usage=RequestUsage(prompt_tokens=10, completion_tokens=5), + cached=False, + ), + AgentResponse(response="Hello", status="success").model_dump_json(), + ], + model_info={ + "function_calling": True, + "vision": True, + "json_output": True, + "family": ModelFamily.GPT_4O, + "structured_output": True, + }, + ) + agent = AssistantAgent( + "test_agent", + model_client=model_client, + output_content_type=AgentResponse, + # reflect_on_tool_use=True, + tools=[ + _pass_function, + _fail_function, + ], + ) + result = await agent.run() + assert len(result.messages) == 3 + assert isinstance(result.messages[0], ToolCallRequestEvent) + assert isinstance(result.messages[1], ToolCallExecutionEvent) + assert isinstance(result.messages[2], StructuredMessage) + assert isinstance(result.messages[2].content, AgentResponse) # type: ignore[reportUnknownMemberType] + assert result.messages[2].content.response == "Hello" + assert result.messages[2].content.status == "success" + + # Test streaming. + agent = AssistantAgent( + "test_agent", + model_client=model_client, + model_client_stream=True, + output_content_type=AgentResponse, + # reflect_on_tool_use=True, + tools=[ + _pass_function, + _fail_function, + ], + ) + model_client.reset() + stream = agent.run_stream() + stream_result: TaskResult | None = None + async for message in stream: + if isinstance(message, TaskResult): + stream_result = message + assert stream_result is not None + assert len(stream_result.messages) == 3 + assert isinstance(stream_result.messages[0], ToolCallRequestEvent) + assert isinstance(stream_result.messages[1], ToolCallExecutionEvent) + assert isinstance(stream_result.messages[2], StructuredMessage) + assert isinstance(stream_result.messages[2].content, AgentResponse) # type: ignore[reportUnknownMemberType] + assert stream_result.messages[2].content.response == "Hello" + assert stream_result.messages[2].content.status == "success" + + # Test when reflect_on_tool_use is False + model_client.reset() + agent = AssistantAgent( + "test_agent", + model_client=model_client, + output_content_type=AgentResponse, + reflect_on_tool_use=False, + tools=[ + _pass_function, + _fail_function, + ], + ) + result = await agent.run() + assert len(result.messages) == 3 + assert isinstance(result.messages[0], ToolCallRequestEvent) + assert isinstance(result.messages[1], ToolCallExecutionEvent) + assert isinstance(result.messages[2], ToolCallSummaryMessage) + + @pytest.mark.asyncio async def test_handoffs() -> None: handoff = Handoff(target="agent2") diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/agents.ipynb b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/agents.ipynb index b05d82b7ccf5..69b5418b115d 100644 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/agents.ipynb +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/agents.ipynb @@ -1,848 +1,862 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Agents\n", - "\n", - "AutoGen AgentChat provides a set of preset Agents, each with variations in how an agent might respond to messages.\n", - "All agents share the following attributes and methods:\n", - "\n", - "- {py:attr}`~autogen_agentchat.agents.BaseChatAgent.name`: The unique name of the agent.\n", - "- {py:attr}`~autogen_agentchat.agents.BaseChatAgent.description`: The description of the agent in text.\n", - "- {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_messages`: Send the agent a sequence of {py:class}`~autogen_agentchat.messages.BaseChatMessage` and get a {py:class}`~autogen_agentchat.base.Response`. **It is important to note that agents are expected to be stateful and this method is expected to be called with new messages, not the complete history**.\n", - "- {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_messages_stream`: Same as {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_messages` but returns an iterator of {py:class}`~autogen_agentchat.messages.BaseAgentEvent` or {py:class}`~autogen_agentchat.messages.BaseChatMessage` followed by a {py:class}`~autogen_agentchat.base.Response` as the last item.\n", - "- {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_reset`: Reset the agent to its initial state.\n", - "- {py:meth}`~autogen_agentchat.agents.BaseChatAgent.run` and {py:meth}`~autogen_agentchat.agents.BaseChatAgent.run_stream`: convenience methods that call {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_messages` and {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_messages_stream` respectively but offer the same interface as [Teams](./teams.ipynb).\n", - "\n", - "See {py:mod}`autogen_agentchat.messages` for more information on AgentChat message types.\n", - "\n", - "\n", - "## Assistant Agent\n", - "\n", - "{py:class}`~autogen_agentchat.agents.AssistantAgent` is a built-in agent that\n", - "uses a language model and has the ability to use tools." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from autogen_agentchat.agents import AssistantAgent\n", - "from autogen_agentchat.messages import TextMessage\n", - "from autogen_agentchat.ui import Console\n", - "from autogen_core import CancellationToken\n", - "from autogen_ext.models.openai import OpenAIChatCompletionClient" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "# Define a tool that searches the web for information.\n", - "async def web_search(query: str) -> str:\n", - " \"\"\"Find information on the web\"\"\"\n", - " return \"AutoGen is a programming framework for building multi-agent applications.\"\n", - "\n", - "\n", - "# Create an agent that uses the OpenAI GPT-4o model.\n", - "model_client = OpenAIChatCompletionClient(\n", - " model=\"gpt-4o\",\n", - " # api_key=\"YOUR_API_KEY\",\n", - ")\n", - "agent = AssistantAgent(\n", - " name=\"assistant\",\n", - " model_client=model_client,\n", - " tools=[web_search],\n", - " system_message=\"Use tools to solve tasks.\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "## Getting Responses\n", - "\n", - "We can use the {py:meth}`~autogen_agentchat.agents.AssistantAgent.on_messages` method to get the agent response to a given message.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[ToolCallRequestEvent(source='assistant', models_usage=RequestUsage(prompt_tokens=598, completion_tokens=16), content=[FunctionCall(id='call_9UWYM1CgE3ZbnJcSJavNDB79', arguments='{\"query\":\"AutoGen\"}', name='web_search')], type='ToolCallRequestEvent'), ToolCallExecutionEvent(source='assistant', models_usage=None, content=[FunctionExecutionResult(content='AutoGen is a programming framework for building multi-agent applications.', call_id='call_9UWYM1CgE3ZbnJcSJavNDB79', is_error=False)], type='ToolCallExecutionEvent')]\n", - "source='assistant' models_usage=None content='AutoGen is a programming framework for building multi-agent applications.' type='ToolCallSummaryMessage'\n" - ] - } - ], - "source": [ - "async def assistant_run() -> None:\n", - " response = await agent.on_messages(\n", - " [TextMessage(content=\"Find information on AutoGen\", source=\"user\")],\n", - " cancellation_token=CancellationToken(),\n", - " )\n", - " print(response.inner_messages)\n", - " print(response.chat_message)\n", - "\n", - "\n", - "# Use asyncio.run(assistant_run()) when running in a script.\n", - "await assistant_run()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The call to the {py:meth}`~autogen_agentchat.agents.AssistantAgent.on_messages` method\n", - "returns a {py:class}`~autogen_agentchat.base.Response`\n", - "that contains the agent's final response in the {py:attr}`~autogen_agentchat.base.Response.chat_message` attribute,\n", - "as well as a list of inner messages in the {py:attr}`~autogen_agentchat.base.Response.inner_messages` attribute,\n", - "which stores the agent's \"thought process\" that led to the final response.\n", - "\n", - "```{note}\n", - "It is important to note that {py:meth}`~autogen_agentchat.agents.AssistantAgent.on_messages`\n", - "will update the internal state of the agent -- it will add the messages to the agent's\n", - "history. So you should call this method with new messages.\n", - "**You should not repeatedly call this method with the same messages or the complete history.**\n", - "```\n", - "\n", - "```{note}\n", - "Unlike in v0.2 AgentChat, the tools are executed by the same agent directly within\n", - "the same call to {py:meth}`~autogen_agentchat.agents.AssistantAgent.on_messages`.\n", - "By default, the agent will return the result of the tool call as the final response.\n", - "```\n", - "\n", - "You can also call the {py:meth}`~autogen_agentchat.agents.BaseChatAgent.run` method, which is a convenience method that calls {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_messages`. \n", - "It follows the same interface as [Teams](./teams.ipynb) and returns a {py:class}`~autogen_agentchat.base.TaskResult` object." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Multi-Modal Input\n", - "\n", - "The {py:class}`~autogen_agentchat.agents.AssistantAgent` can handle multi-modal input\n", - "by providing the input as a {py:class}`~autogen_agentchat.messages.MultiModalMessage`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from io import BytesIO\n", - "\n", - "import PIL\n", - "import requests\n", - "from autogen_agentchat.messages import MultiModalMessage\n", - "from autogen_core import Image\n", - "\n", - "# Create a multi-modal message with random image and text.\n", - "pil_image = PIL.Image.open(BytesIO(requests.get(\"https://picsum.photos/300/200\").content))\n", - "img = Image(pil_image)\n", - "multi_modal_message = MultiModalMessage(content=[\"Can you describe the content of this image?\", img], source=\"user\")\n", - "img" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The image depicts a vintage car, likely from the 1930s or 1940s, with a sleek, classic design. The car seems to be customized or well-maintained, as indicated by its shiny exterior and lowered stance. It has a prominent grille and round headlights. There's a license plate on the front with the text \"FARMER BOY.\" The setting appears to be a street with old-style buildings in the background, suggesting a historical or retro theme.\n" - ] - } - ], - "source": [ - "# Use asyncio.run(...) when running in a script.\n", - "response = await agent.on_messages([multi_modal_message], CancellationToken())\n", - "print(response.chat_message)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can also use {py:class}`~autogen_agentchat.messages.MultiModalMessage` as a `task`\n", - "input to the {py:meth}`~autogen_agentchat.agents.BaseChatAgent.run` method." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Streaming Messages\n", - "\n", - "We can also stream each message as it is generated by the agent by using the\n", - "{py:meth}`~autogen_agentchat.agents.AssistantAgent.on_messages_stream` method,\n", - "and use {py:class}`~autogen_agentchat.ui.Console` to print the messages\n", - "as they appear to the console." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "---------- assistant ----------\n", - "[FunctionCall(id='call_fSp5iTGVm2FKw5NIvfECSqNd', arguments='{\"query\":\"AutoGen information\"}', name='web_search')]\n", - "[Prompt tokens: 61, Completion tokens: 16]\n", - "---------- assistant ----------\n", - "[FunctionExecutionResult(content='AutoGen is a programming framework for building multi-agent applications.', call_id='call_fSp5iTGVm2FKw5NIvfECSqNd')]\n", - "---------- assistant ----------\n", - "AutoGen is a programming framework designed for building multi-agent applications. If you need more detailed information or specific aspects about AutoGen, feel free to ask!\n", - "[Prompt tokens: 93, Completion tokens: 32]\n", - "---------- Summary ----------\n", - "Number of inner messages: 2\n", - "Total prompt tokens: 154\n", - "Total completion tokens: 48\n", - "Duration: 4.30 seconds\n" - ] - } - ], - "source": [ - "async def assistant_run_stream() -> None:\n", - " # Option 1: read each message from the stream (as shown in the previous example).\n", - " # async for message in agent.on_messages_stream(\n", - " # [TextMessage(content=\"Find information on AutoGen\", source=\"user\")],\n", - " # cancellation_token=CancellationToken(),\n", - " # ):\n", - " # print(message)\n", - "\n", - " # Option 2: use Console to print all messages as they appear.\n", - " await Console(\n", - " agent.on_messages_stream(\n", - " [TextMessage(content=\"Find information on AutoGen\", source=\"user\")],\n", - " cancellation_token=CancellationToken(),\n", - " ),\n", - " output_stats=True, # Enable stats printing.\n", - " )\n", - "\n", - "\n", - "# Use asyncio.run(assistant_run_stream()) when running in a script.\n", - "await assistant_run_stream()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The {py:meth}`~autogen_agentchat.agents.AssistantAgent.on_messages_stream` method\n", - "returns an asynchronous generator that yields each inner message generated by the agent,\n", - "with the final item being the response message in the {py:attr}`~autogen_agentchat.base.Response.chat_message` attribute.\n", - "\n", - "From the messages, you can observe that the assistant agent utilized the `web_search` tool to\n", - "gather information and responded based on the search results.\n", - "\n", - "You can also use {py:meth}`~autogen_agentchat.agents.BaseChatAgent.run_stream` to get the same streaming behavior as {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_messages_stream`. It follows the same interface as [Teams](./teams.ipynb)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using Tools\n", - "\n", - "Large Language Models (LLMs) are typically limited to generating text or code responses. \n", - "However, many complex tasks benefit from the ability to use external tools that perform specific actions,\n", - "such as fetching data from APIs or databases.\n", - "\n", - "To address this limitation, modern LLMs can now accept a list of available tool schemas \n", - "(descriptions of tools and their arguments) and generate a tool call message. \n", - "This capability is known as **Tool Calling** or **Function Calling** and \n", - "is becoming a popular pattern in building intelligent agent-based applications.\n", - "Refer to the documentation from [OpenAI](https://platform.openai.com/docs/guides/function-calling) \n", - "and [Anthropic](https://docs.anthropic.com/en/docs/build-with-claude/tool-use) for more information about tool calling in LLMs.\n", - "\n", - "In AgentChat, the {py:class}`~autogen_agentchat.agents.AssistantAgent` can use tools to perform specific actions.\n", - "The `web_search` tool is one such tool that allows the assistant agent to search the web for information.\n", - "A custom tool can be a Python function or a subclass of the {py:class}`~autogen_core.tools.BaseTool`.\n", - "\n", - "```{note}\n", - "For how to use model clients directly with tools, refer to the [Tools](../../core-user-guide/components/tools.ipynb) section\n", - "in the Core User Guide.\n", - "```\n", - "\n", - "By default, when {py:class}`~autogen_agentchat.agents.AssistantAgent` executes a tool,\n", - "it will return the tool's output as a string in {py:class}`~autogen_agentchat.messages.ToolCallSummaryMessage` in its response.\n", - "If your tool does not return a well-formed string in natural language, you\n", - "can add a reflection step to have the model summarize the tool's output,\n", - "by setting the `reflect_on_tool_use=True` parameter in the {py:class}`~autogen_agentchat.agents.AssistantAgent` constructor.\n", - "\n", - "### Built-in Tools\n", - "\n", - "AutoGen Extension provides a set of built-in tools that can be used with the Assistant Agent.\n", - "Head over to the [API documentation](../../../reference/index.md) for all the available tools\n", - "under the `autogen_ext.tools` namespace. For example, you can find the following tools:\n", - "\n", - "- {py:mod}`~autogen_ext.tools.graphrag`: Tools for using GraphRAG index.\n", - "- {py:mod}`~autogen_ext.tools.http`: Tools for making HTTP requests.\n", - "- {py:mod}`~autogen_ext.tools.langchain`: Adaptor for using LangChain tools.\n", - "- {py:mod}`~autogen_ext.tools.mcp`: Tools for using Model Chat Protocol (MCP) servers." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Function Tool\n", - "\n", - "The {py:class}`~autogen_agentchat.agents.AssistantAgent` automatically\n", - "converts a Python function into a {py:class}`~autogen_core.tools.FunctionTool`\n", - "which can be used as a tool by the agent and automatically generates the tool schema\n", - "from the function signature and docstring.\n", - "\n", - "The `web_search_func` tool is an example of a function tool.\n", - "The schema is automatically generated." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'name': 'web_search_func',\n", - " 'description': 'Find information on the web',\n", - " 'parameters': {'type': 'object',\n", - " 'properties': {'query': {'description': 'query',\n", - " 'title': 'Query',\n", - " 'type': 'string'}},\n", - " 'required': ['query'],\n", - " 'additionalProperties': False},\n", - " 'strict': False}" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from autogen_core.tools import FunctionTool\n", - "\n", - "\n", - "# Define a tool using a Python function.\n", - "async def web_search_func(query: str) -> str:\n", - " \"\"\"Find information on the web\"\"\"\n", - " return \"AutoGen is a programming framework for building multi-agent applications.\"\n", - "\n", - "\n", - "# This step is automatically performed inside the AssistantAgent if the tool is a Python function.\n", - "web_search_function_tool = FunctionTool(web_search_func, description=\"Find information on the web\")\n", - "# The schema is provided to the model during AssistantAgent's on_messages call.\n", - "web_search_function_tool.schema" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Model Context Protocol Tools\n", - "\n", - "The {py:class}`~autogen_agentchat.agents.AssistantAgent` can also use tools that are\n", - "served from a Model Context Protocol (MCP) server\n", - "using {py:func}`~autogen_ext.tools.mcp.mcp_server_tools`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Seattle, located in Washington state, is the most populous city in the state and a major city in the Pacific Northwest region of the United States. It's known for its vibrant cultural scene, significant economic presence, and rich history. Here are some key points about Seattle from the Wikipedia page:\n", - "\n", - "1. **History and Geography**: Seattle is situated between Puget Sound and Lake Washington, with the Cascade Range to the east and the Olympic Mountains to the west. Its history is deeply rooted in Native American heritage and its development was accelerated with the arrival of settlers in the 19th century. The city was officially incorporated in 1869.\n", - "\n", - "2. **Economy**: Seattle is a major economic hub with a diverse economy anchored by sectors like aerospace, technology, and retail. It's home to influential companies such as Amazon and Starbucks, and has a significant impact on the tech industry due to companies like Microsoft and other technology enterprises in the surrounding area.\n", - "\n", - "3. **Cultural Significance**: Known for its music scene, Seattle was the birthplace of grunge music in the early 1990s. It also boasts significant attractions like the Space Needle, Pike Place Market, and the Seattle Art Museum. \n", - "\n", - "4. **Education and Innovation**: The city hosts important educational institutions, with the University of Washington being a leading research university. Seattle is recognized for fostering innovation and is a leader in environmental sustainability efforts.\n", - "\n", - "5. **Demographics and Diversity**: Seattle is noted for its diverse population, reflected in its rich cultural tapestry. It has seen a significant increase in population, leading to urban development and changes in its social landscape.\n", - "\n", - "These points highlight Seattle as a dynamic city with a significant cultural, economic, and educational influence within the United States and beyond.\n" - ] - } - ], - "source": [ - "from autogen_agentchat.agents import AssistantAgent\n", - "from autogen_ext.models.openai import OpenAIChatCompletionClient\n", - "from autogen_ext.tools.mcp import StdioServerParams, mcp_server_tools\n", - "\n", - "# Get the fetch tool from mcp-server-fetch.\n", - "fetch_mcp_server = StdioServerParams(command=\"uvx\", args=[\"mcp-server-fetch\"])\n", - "tools = await mcp_server_tools(fetch_mcp_server)\n", - "\n", - "# Create an agent that can use the fetch tool.\n", - "model_client = OpenAIChatCompletionClient(model=\"gpt-4o\")\n", - "agent = AssistantAgent(name=\"fetcher\", model_client=model_client, tools=tools, reflect_on_tool_use=True) # type: ignore\n", - "\n", - "# Let the agent fetch the content of a URL and summarize it.\n", - "result = await agent.run(task=\"Summarize the content of https://en.wikipedia.org/wiki/Seattle\")\n", - "assert isinstance(result.messages[-1], TextMessage)\n", - "print(result.messages[-1].content)\n", - "\n", - "# Close the connection to the model client.\n", - "await model_client.close()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Langchain Tools\n", - "\n", - "You can also use tools from the Langchain library\n", - "by wrapping them in {py:class}`~autogen_ext.tools.langchain.LangChainToolAdapter`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "---------- assistant ----------\n", - "[FunctionCall(id='call_BEYRkf53nBS1G2uG60wHP0zf', arguments='{\"query\":\"df[\\'Age\\'].mean()\"}', name='python_repl_ast')]\n", - "[Prompt tokens: 111, Completion tokens: 22]\n", - "---------- assistant ----------\n", - "[FunctionExecutionResult(content='29.69911764705882', call_id='call_BEYRkf53nBS1G2uG60wHP0zf')]\n", - "---------- assistant ----------\n", - "29.69911764705882\n", - "---------- Summary ----------\n", - "Number of inner messages: 2\n", - "Total prompt tokens: 111\n", - "Total completion tokens: 22\n", - "Duration: 0.62 seconds\n" - ] - }, - { - "data": { - "text/plain": [ - "Response(chat_message=ToolCallSummaryMessage(source='assistant', models_usage=None, content='29.69911764705882', type='ToolCallSummaryMessage'), inner_messages=[ToolCallRequestEvent(source='assistant', models_usage=RequestUsage(prompt_tokens=111, completion_tokens=22), content=[FunctionCall(id='call_BEYRkf53nBS1G2uG60wHP0zf', arguments='{\"query\":\"df[\\'Age\\'].mean()\"}', name='python_repl_ast')], type='ToolCallRequestEvent'), ToolCallExecutionEvent(source='assistant', models_usage=None, content=[FunctionExecutionResult(content='29.69911764705882', call_id='call_BEYRkf53nBS1G2uG60wHP0zf')], type='ToolCallExecutionEvent')])" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import pandas as pd\n", - "from autogen_ext.tools.langchain import LangChainToolAdapter\n", - "from langchain_experimental.tools.python.tool import PythonAstREPLTool\n", - "\n", - "df = pd.read_csv(\"https://raw.githubusercontent.com/pandas-dev/pandas/main/doc/data/titanic.csv\")\n", - "tool = LangChainToolAdapter(PythonAstREPLTool(locals={\"df\": df}))\n", - "model_client = OpenAIChatCompletionClient(model=\"gpt-4o\")\n", - "agent = AssistantAgent(\n", - " \"assistant\", tools=[tool], model_client=model_client, system_message=\"Use the `df` variable to access the dataset.\"\n", - ")\n", - "await Console(\n", - " agent.on_messages_stream(\n", - " [TextMessage(content=\"What's the average age of the passengers?\", source=\"user\")], CancellationToken()\n", - " ),\n", - " output_stats=True,\n", - ")\n", - "\n", - "await model_client.close()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Parallel Tool Calls\n", - "\n", - "Some models support parallel tool calls, which can be useful for tasks that require multiple tools to be called simultaneously.\n", - "By default, if the model client produces multiple tool calls, {py:class}`~autogen_agentchat.agents.AssistantAgent`\n", - "will call the tools in parallel.\n", - "\n", - "You may want to disable parallel tool calls when the tools have side effects that may interfere with each other, or,\n", - "when agent behavior needs to be consistent across different models.\n", - "This should be done at the model client level.\n", - "\n", - "For {py:class}`~autogen_ext.models.openai.OpenAIChatCompletionClient` and {py:class}`~autogen_ext.models.openai.AzureOpenAIChatCompletionClient`,\n", - "set `parallel_tool_calls=False` to disable parallel tool calls." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model_client_no_parallel_tool_call = OpenAIChatCompletionClient(\n", - " model=\"gpt-4o\",\n", - " parallel_tool_calls=False, # type: ignore\n", - ")\n", - "agent_no_parallel_tool_call = AssistantAgent(\n", - " name=\"assistant\",\n", - " model_client=model_client_no_parallel_tool_call,\n", - " tools=[web_search],\n", - " system_message=\"Use tools to solve tasks.\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Running an Agent in a Loop\n", - "\n", - "The {py:class}`~autogen_agentchat.agents.AssistantAgent` executes one\n", - "step at a time: one model call, followed by one tool call (or parallel tool calls), and then\n", - "an optional reflection.\n", - "\n", - "To run it in a loop, for example, running it until it stops producing\n", - "tool calls, please refer to [Single-Agent Team](./teams.ipynb#single-agent-team)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Structured Output\n", - "\n", - "Structured output allows models to return structured JSON text with pre-defined schema\n", - "provided by the application. Different from JSON-mode, the schema can be provided\n", - "as a [Pydantic BaseModel](https://docs.pydantic.dev/latest/concepts/models/)\n", - "class, which can also be used to validate the output. \n", - "\n", - "```{note}\n", - "Structured output is only available for models that support it. It also\n", - "requires the model client to support structured output as well.\n", - "Currently, the {py:class}`~autogen_ext.models.openai.OpenAIChatCompletionClient`\n", - "and {py:class}`~autogen_ext.models.openai.AzureOpenAIChatCompletionClient`\n", - "support structured output.\n", - "```\n", - "\n", - "Structured output is also useful for incorporating Chain-of-Thought\n", - "reasoning in the agent's responses.\n", - "See the example below for how to use structured output with the assistant agent." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "---------- user ----------\n", - "I am happy.\n", - "---------- assistant ----------\n", - "{\"thoughts\":\"The user explicitly states that they are happy.\",\"response\":\"happy\"}\n" - ] - }, - { - "data": { - "text/plain": [ - "TaskResult(messages=[TextMessage(source='user', models_usage=None, content='I am happy.', type='TextMessage'), TextMessage(source='assistant', models_usage=RequestUsage(prompt_tokens=89, completion_tokens=18), content='{\"thoughts\":\"The user explicitly states that they are happy.\",\"response\":\"happy\"}', type='TextMessage')], stop_reason=None)" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from typing import Literal\n", - "\n", - "from pydantic import BaseModel\n", - "\n", - "\n", - "# The response format for the agent as a Pydantic base model.\n", - "class AgentResponse(BaseModel):\n", - " thoughts: str\n", - " response: Literal[\"happy\", \"sad\", \"neutral\"]\n", - "\n", - "\n", - "# Create an agent that uses the OpenAI GPT-4o model with the custom response format.\n", - "model_client = OpenAIChatCompletionClient(\n", - " model=\"gpt-4o\",\n", - " response_format=AgentResponse, # type: ignore\n", - ")\n", - "agent = AssistantAgent(\n", - " \"assistant\",\n", - " model_client=model_client,\n", - " system_message=\"Categorize the input as happy, sad, or neutral following the JSON format.\",\n", - ")\n", - "\n", - "await Console(agent.run_stream(task=\"I am happy.\"))\n", - "await model_client.close()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Streaming Tokens\n", - "\n", - "You can stream the tokens generated by the model client by setting `model_client_stream=True`.\n", - "This will cause the agent to yield {py:class}`~autogen_agentchat.messages.ModelClientStreamingChunkEvent` messages\n", - "in {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_messages_stream` and {py:meth}`~autogen_agentchat.agents.BaseChatAgent.run_stream`.\n", - "\n", - "The underlying model API must support streaming tokens for this to work.\n", - "Please check with your model provider to see if this is supported." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "source='assistant' models_usage=None content='Two' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' cities' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' in' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' South' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' America' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' are' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' Buenos' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' Aires' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' in' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' Argentina' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' and' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' São' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' Paulo' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' in' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' Brazil' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content='.' type='ModelClientStreamingChunkEvent'\n", - "Response(chat_message=TextMessage(source='assistant', models_usage=RequestUsage(prompt_tokens=0, completion_tokens=0), content='Two cities in South America are Buenos Aires in Argentina and São Paulo in Brazil.', type='TextMessage'), inner_messages=[])\n" - ] - } - ], - "source": [ - "model_client = OpenAIChatCompletionClient(model=\"gpt-4o\")\n", - "\n", - "streaming_assistant = AssistantAgent(\n", - " name=\"assistant\",\n", - " model_client=model_client,\n", - " system_message=\"You are a helpful assistant.\",\n", - " model_client_stream=True, # Enable streaming tokens.\n", - ")\n", - "\n", - "# Use an async function and asyncio.run() in a script.\n", - "async for message in streaming_assistant.on_messages_stream( # type: ignore\n", - " [TextMessage(content=\"Name two cities in South America\", source=\"user\")],\n", - " cancellation_token=CancellationToken(),\n", - "):\n", - " print(message)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can see the streaming chunks in the output above.\n", - "The chunks are generated by the model client and are yielded by the agent as they are received.\n", - "The final response, the concatenation of all the chunks, is yielded right after the last chunk.\n", - "\n", - "Similarly, {py:meth}`~autogen_agentchat.agents.BaseChatAgent.run_stream` will also yield the same streaming chunks,\n", - "followed by a full text message right after the last chunk." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "source='user' models_usage=None content='Name two cities in North America.' type='TextMessage'\n", - "source='assistant' models_usage=None content='Two' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' cities' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' in' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' North' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' America' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' are' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' New' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' York' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' City' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' in' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' the' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' United' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' States' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' and' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' Toronto' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' in' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content=' Canada' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=None content='.' type='ModelClientStreamingChunkEvent'\n", - "source='assistant' models_usage=RequestUsage(prompt_tokens=0, completion_tokens=0) content='Two cities in North America are New York City in the United States and Toronto in Canada.' type='TextMessage'\n", - "TaskResult(messages=[TextMessage(source='user', models_usage=None, content='Name two cities in North America.', type='TextMessage'), TextMessage(source='assistant', models_usage=RequestUsage(prompt_tokens=0, completion_tokens=0), content='Two cities in North America are New York City in the United States and Toronto in Canada.', type='TextMessage')], stop_reason=None)\n" - ] - } - ], - "source": [ - "async for message in streaming_assistant.run_stream(task=\"Name two cities in North America.\"): # type: ignore\n", - " print(message)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using Model Context\n", - "\n", - "{py:class}`~autogen_agentchat.agents.AssistantAgent` has a `model_context`\n", - "parameter that can be used to pass in a {py:class}`~autogen_core.model_context.ChatCompletionContext`\n", - "object. This allows the agent to use different model contexts, such as\n", - "{py:class}`~autogen_core.model_context.BufferedChatCompletionContext` to\n", - "limit the context sent to the model.\n", - "\n", - "By default, {py:class}`~autogen_agentchat.agents.AssistantAgent` uses\n", - "the {py:class}`~autogen_core.model_context.UnboundedChatCompletionContext`\n", - "which sends the full conversation history to the model. To limit the context\n", - "to the last `n` messages, you can use the {py:class}`~autogen_core.model_context.BufferedChatCompletionContext`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from autogen_core.model_context import BufferedChatCompletionContext\n", - "\n", - "# Create an agent that uses only the last 5 messages in the context to generate responses.\n", - "agent = AssistantAgent(\n", - " name=\"assistant\",\n", - " model_client=model_client,\n", - " tools=[web_search],\n", - " system_message=\"Use tools to solve tasks.\",\n", - " model_context=BufferedChatCompletionContext(buffer_size=5), # Only use the last 5 messages in the context.\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Other Preset Agents\n", - "\n", - "The following preset agents are available:\n", - "\n", - "- {py:class}`~autogen_agentchat.agents.UserProxyAgent`: An agent that takes user input returns it as responses.\n", - "- {py:class}`~autogen_agentchat.agents.CodeExecutorAgent`: An agent that can execute code.\n", - "- {py:class}`~autogen_ext.agents.openai.OpenAIAssistantAgent`: An agent that is backed by an OpenAI Assistant, with ability to use custom tools.\n", - "- {py:class}`~autogen_ext.agents.web_surfer.MultimodalWebSurfer`: A multi-modal agent that can search the web and visit web pages for information.\n", - "- {py:class}`~autogen_ext.agents.file_surfer.FileSurfer`: An agent that can search and browse local files for information.\n", - "- {py:class}`~autogen_ext.agents.video_surfer.VideoSurfer`: An agent that can watch videos for information." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Next Step\n", - "\n", - "Having explored the usage of the {py:class}`~autogen_agentchat.agents.AssistantAgent`, we can now proceed to the next section to learn about the teams feature in AgentChat.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.9" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Agents\n", + "\n", + "AutoGen AgentChat provides a set of preset Agents, each with variations in how an agent might respond to messages.\n", + "All agents share the following attributes and methods:\n", + "\n", + "- {py:attr}`~autogen_agentchat.agents.BaseChatAgent.name`: The unique name of the agent.\n", + "- {py:attr}`~autogen_agentchat.agents.BaseChatAgent.description`: The description of the agent in text.\n", + "- {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_messages`: Send the agent a sequence of messages that subclass {py:class}`~autogen_agentchat.messages.BaseChatMessage` and get a {py:class}`~autogen_agentchat.base.Response`. **It is important to note that agents are expected to be stateful and this method is expected to be called with new messages, not the complete history**.\n", + "- {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_messages_stream`: Same as {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_messages` but returns an iterator of messages that subclass {py:class}`~autogen_agentchat.messages.BaseAgentEvent` or {py:class}`~autogen_agentchat.messages.BaseChatMessage` followed by a {py:class}`~autogen_agentchat.base.Response` as the last item.\n", + "- {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_reset`: Reset the agent to its initial state.\n", + "- {py:meth}`~autogen_agentchat.agents.BaseChatAgent.run` and {py:meth}`~autogen_agentchat.agents.BaseChatAgent.run_stream`: convenience methods that call {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_messages` and {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_messages_stream` respectively but offer the same interface as [Teams](./teams.ipynb).\n", + "\n", + "See {py:mod}`autogen_agentchat.messages` for more information on AgentChat message types.\n", + "\n", + "\n", + "## Assistant Agent\n", + "\n", + "{py:class}`~autogen_agentchat.agents.AssistantAgent` is a built-in agent that\n", + "uses a language model and has the ability to use tools." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from autogen_agentchat.agents import AssistantAgent\n", + "from autogen_agentchat.messages import StructuredMessage, TextMessage\n", + "from autogen_agentchat.ui import Console\n", + "from autogen_core import CancellationToken\n", + "from autogen_ext.models.openai import OpenAIChatCompletionClient" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "# Define a tool that searches the web for information.\n", + "async def web_search(query: str) -> str:\n", + " \"\"\"Find information on the web\"\"\"\n", + " return \"AutoGen is a programming framework for building multi-agent applications.\"\n", + "\n", + "\n", + "# Create an agent that uses the OpenAI GPT-4o model.\n", + "model_client = OpenAIChatCompletionClient(\n", + " model=\"gpt-4o\",\n", + " # api_key=\"YOUR_API_KEY\",\n", + ")\n", + "agent = AssistantAgent(\n", + " name=\"assistant\",\n", + " model_client=model_client,\n", + " tools=[web_search],\n", + " system_message=\"Use tools to solve tasks.\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Getting Responses\n", + "\n", + "We can use the {py:meth}`~autogen_agentchat.agents.AssistantAgent.on_messages` method to get the agent response to a given message.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[ToolCallRequestEvent(source='assistant', models_usage=RequestUsage(prompt_tokens=598, completion_tokens=16), content=[FunctionCall(id='call_9UWYM1CgE3ZbnJcSJavNDB79', arguments='{\"query\":\"AutoGen\"}', name='web_search')], type='ToolCallRequestEvent'), ToolCallExecutionEvent(source='assistant', models_usage=None, content=[FunctionExecutionResult(content='AutoGen is a programming framework for building multi-agent applications.', call_id='call_9UWYM1CgE3ZbnJcSJavNDB79', is_error=False)], type='ToolCallExecutionEvent')]\n", + "source='assistant' models_usage=None content='AutoGen is a programming framework for building multi-agent applications.' type='ToolCallSummaryMessage'\n" + ] + } + ], + "source": [ + "async def assistant_run() -> None:\n", + " response = await agent.on_messages(\n", + " [TextMessage(content=\"Find information on AutoGen\", source=\"user\")],\n", + " cancellation_token=CancellationToken(),\n", + " )\n", + " print(response.inner_messages)\n", + " print(response.chat_message)\n", + "\n", + "\n", + "# Use asyncio.run(assistant_run()) when running in a script.\n", + "await assistant_run()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The call to the {py:meth}`~autogen_agentchat.agents.AssistantAgent.on_messages` method\n", + "returns a {py:class}`~autogen_agentchat.base.Response`\n", + "that contains the agent's final response in the {py:attr}`~autogen_agentchat.base.Response.chat_message` attribute,\n", + "as well as a list of inner messages in the {py:attr}`~autogen_agentchat.base.Response.inner_messages` attribute,\n", + "which stores the agent's \"thought process\" that led to the final response.\n", + "\n", + "```{note}\n", + "It is important to note that {py:meth}`~autogen_agentchat.agents.AssistantAgent.on_messages`\n", + "will update the internal state of the agent -- it will add the messages to the agent's\n", + "history. So you should call this method with new messages.\n", + "**You should not repeatedly call this method with the same messages or the complete history.**\n", + "```\n", + "\n", + "```{note}\n", + "Unlike in v0.2 AgentChat, the tools are executed by the same agent directly within\n", + "the same call to {py:meth}`~autogen_agentchat.agents.AssistantAgent.on_messages`.\n", + "By default, the agent will return the result of the tool call as the final response.\n", + "```\n", + "\n", + "You can also call the {py:meth}`~autogen_agentchat.agents.BaseChatAgent.run` method, which is a convenience method that calls {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_messages`. \n", + "It follows the same interface as [Teams](./teams.ipynb) and returns a {py:class}`~autogen_agentchat.base.TaskResult` object." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi-Modal Input\n", + "\n", + "The {py:class}`~autogen_agentchat.agents.AssistantAgent` can handle multi-modal input\n", + "by providing the input as a {py:class}`~autogen_agentchat.messages.MultiModalMessage`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from io import BytesIO\n", + "\n", + "import PIL\n", + "import requests\n", + "from autogen_agentchat.messages import MultiModalMessage\n", + "from autogen_core import Image\n", + "\n", + "# Create a multi-modal message with random image and text.\n", + "pil_image = PIL.Image.open(BytesIO(requests.get(\"https://picsum.photos/300/200\").content))\n", + "img = Image(pil_image)\n", + "multi_modal_message = MultiModalMessage(content=[\"Can you describe the content of this image?\", img], source=\"user\")\n", + "img" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The image depicts a vintage car, likely from the 1930s or 1940s, with a sleek, classic design. The car seems to be customized or well-maintained, as indicated by its shiny exterior and lowered stance. It has a prominent grille and round headlights. There's a license plate on the front with the text \"FARMER BOY.\" The setting appears to be a street with old-style buildings in the background, suggesting a historical or retro theme.\n" + ] + } + ], + "source": [ + "# Use asyncio.run(...) when running in a script.\n", + "response = await agent.on_messages([multi_modal_message], CancellationToken())\n", + "print(response.chat_message)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also use {py:class}`~autogen_agentchat.messages.MultiModalMessage` as a `task`\n", + "input to the {py:meth}`~autogen_agentchat.agents.BaseChatAgent.run` method." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Streaming Messages\n", + "\n", + "We can also stream each message as it is generated by the agent by using the\n", + "{py:meth}`~autogen_agentchat.agents.AssistantAgent.on_messages_stream` method,\n", + "and use {py:class}`~autogen_agentchat.ui.Console` to print the messages\n", + "as they appear to the console." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---------- assistant ----------\n", + "[FunctionCall(id='call_fSp5iTGVm2FKw5NIvfECSqNd', arguments='{\"query\":\"AutoGen information\"}', name='web_search')]\n", + "[Prompt tokens: 61, Completion tokens: 16]\n", + "---------- assistant ----------\n", + "[FunctionExecutionResult(content='AutoGen is a programming framework for building multi-agent applications.', call_id='call_fSp5iTGVm2FKw5NIvfECSqNd')]\n", + "---------- assistant ----------\n", + "AutoGen is a programming framework designed for building multi-agent applications. If you need more detailed information or specific aspects about AutoGen, feel free to ask!\n", + "[Prompt tokens: 93, Completion tokens: 32]\n", + "---------- Summary ----------\n", + "Number of inner messages: 2\n", + "Total prompt tokens: 154\n", + "Total completion tokens: 48\n", + "Duration: 4.30 seconds\n" + ] + } + ], + "source": [ + "async def assistant_run_stream() -> None:\n", + " # Option 1: read each message from the stream (as shown in the previous example).\n", + " # async for message in agent.on_messages_stream(\n", + " # [TextMessage(content=\"Find information on AutoGen\", source=\"user\")],\n", + " # cancellation_token=CancellationToken(),\n", + " # ):\n", + " # print(message)\n", + "\n", + " # Option 2: use Console to print all messages as they appear.\n", + " await Console(\n", + " agent.on_messages_stream(\n", + " [TextMessage(content=\"Find information on AutoGen\", source=\"user\")],\n", + " cancellation_token=CancellationToken(),\n", + " ),\n", + " output_stats=True, # Enable stats printing.\n", + " )\n", + "\n", + "\n", + "# Use asyncio.run(assistant_run_stream()) when running in a script.\n", + "await assistant_run_stream()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The {py:meth}`~autogen_agentchat.agents.AssistantAgent.on_messages_stream` method\n", + "returns an asynchronous generator that yields each inner message generated by the agent,\n", + "with the final item being the response message in the {py:attr}`~autogen_agentchat.base.Response.chat_message` attribute.\n", + "\n", + "From the messages, you can observe that the assistant agent utilized the `web_search` tool to\n", + "gather information and responded based on the search results.\n", + "\n", + "You can also use {py:meth}`~autogen_agentchat.agents.BaseChatAgent.run_stream` to get the same streaming behavior as {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_messages_stream`. It follows the same interface as [Teams](./teams.ipynb)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Tools\n", + "\n", + "Large Language Models (LLMs) are typically limited to generating text or code responses. \n", + "However, many complex tasks benefit from the ability to use external tools that perform specific actions,\n", + "such as fetching data from APIs or databases.\n", + "\n", + "To address this limitation, modern LLMs can now accept a list of available tool schemas \n", + "(descriptions of tools and their arguments) and generate a tool call message. \n", + "This capability is known as **Tool Calling** or **Function Calling** and \n", + "is becoming a popular pattern in building intelligent agent-based applications.\n", + "Refer to the documentation from [OpenAI](https://platform.openai.com/docs/guides/function-calling) \n", + "and [Anthropic](https://docs.anthropic.com/en/docs/build-with-claude/tool-use) for more information about tool calling in LLMs.\n", + "\n", + "In AgentChat, the {py:class}`~autogen_agentchat.agents.AssistantAgent` can use tools to perform specific actions.\n", + "The `web_search` tool is one such tool that allows the assistant agent to search the web for information.\n", + "A custom tool can be a Python function or a subclass of the {py:class}`~autogen_core.tools.BaseTool`.\n", + "\n", + "```{note}\n", + "For how to use model clients directly with tools, refer to the [Tools](../../core-user-guide/components/tools.ipynb) section\n", + "in the Core User Guide.\n", + "```\n", + "\n", + "By default, when {py:class}`~autogen_agentchat.agents.AssistantAgent` executes a tool,\n", + "it will return the tool's output as a string in {py:class}`~autogen_agentchat.messages.ToolCallSummaryMessage` in its response.\n", + "If your tool does not return a well-formed string in natural language, you\n", + "can add a reflection step to have the model summarize the tool's output,\n", + "by setting the `reflect_on_tool_use=True` parameter in the {py:class}`~autogen_agentchat.agents.AssistantAgent` constructor.\n", + "\n", + "### Built-in Tools\n", + "\n", + "AutoGen Extension provides a set of built-in tools that can be used with the Assistant Agent.\n", + "Head over to the [API documentation](../../../reference/index.md) for all the available tools\n", + "under the `autogen_ext.tools` namespace. For example, you can find the following tools:\n", + "\n", + "- {py:mod}`~autogen_ext.tools.graphrag`: Tools for using GraphRAG index.\n", + "- {py:mod}`~autogen_ext.tools.http`: Tools for making HTTP requests.\n", + "- {py:mod}`~autogen_ext.tools.langchain`: Adaptor for using LangChain tools.\n", + "- {py:mod}`~autogen_ext.tools.mcp`: Tools for using Model Chat Protocol (MCP) servers." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Function Tool\n", + "\n", + "The {py:class}`~autogen_agentchat.agents.AssistantAgent` automatically\n", + "converts a Python function into a {py:class}`~autogen_core.tools.FunctionTool`\n", + "which can be used as a tool by the agent and automatically generates the tool schema\n", + "from the function signature and docstring.\n", + "\n", + "The `web_search_func` tool is an example of a function tool.\n", + "The schema is automatically generated." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'name': 'web_search_func',\n", + " 'description': 'Find information on the web',\n", + " 'parameters': {'type': 'object',\n", + " 'properties': {'query': {'description': 'query',\n", + " 'title': 'Query',\n", + " 'type': 'string'}},\n", + " 'required': ['query'],\n", + " 'additionalProperties': False},\n", + " 'strict': False}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from autogen_core.tools import FunctionTool\n", + "\n", + "\n", + "# Define a tool using a Python function.\n", + "async def web_search_func(query: str) -> str:\n", + " \"\"\"Find information on the web\"\"\"\n", + " return \"AutoGen is a programming framework for building multi-agent applications.\"\n", + "\n", + "\n", + "# This step is automatically performed inside the AssistantAgent if the tool is a Python function.\n", + "web_search_function_tool = FunctionTool(web_search_func, description=\"Find information on the web\")\n", + "# The schema is provided to the model during AssistantAgent's on_messages call.\n", + "web_search_function_tool.schema" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Model Context Protocol Tools\n", + "\n", + "The {py:class}`~autogen_agentchat.agents.AssistantAgent` can also use tools that are\n", + "served from a Model Context Protocol (MCP) server\n", + "using {py:func}`~autogen_ext.tools.mcp.mcp_server_tools`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Seattle, located in Washington state, is the most populous city in the state and a major city in the Pacific Northwest region of the United States. It's known for its vibrant cultural scene, significant economic presence, and rich history. Here are some key points about Seattle from the Wikipedia page:\n", + "\n", + "1. **History and Geography**: Seattle is situated between Puget Sound and Lake Washington, with the Cascade Range to the east and the Olympic Mountains to the west. Its history is deeply rooted in Native American heritage and its development was accelerated with the arrival of settlers in the 19th century. The city was officially incorporated in 1869.\n", + "\n", + "2. **Economy**: Seattle is a major economic hub with a diverse economy anchored by sectors like aerospace, technology, and retail. It's home to influential companies such as Amazon and Starbucks, and has a significant impact on the tech industry due to companies like Microsoft and other technology enterprises in the surrounding area.\n", + "\n", + "3. **Cultural Significance**: Known for its music scene, Seattle was the birthplace of grunge music in the early 1990s. It also boasts significant attractions like the Space Needle, Pike Place Market, and the Seattle Art Museum. \n", + "\n", + "4. **Education and Innovation**: The city hosts important educational institutions, with the University of Washington being a leading research university. Seattle is recognized for fostering innovation and is a leader in environmental sustainability efforts.\n", + "\n", + "5. **Demographics and Diversity**: Seattle is noted for its diverse population, reflected in its rich cultural tapestry. It has seen a significant increase in population, leading to urban development and changes in its social landscape.\n", + "\n", + "These points highlight Seattle as a dynamic city with a significant cultural, economic, and educational influence within the United States and beyond.\n" + ] + } + ], + "source": [ + "from autogen_agentchat.agents import AssistantAgent\n", + "from autogen_ext.models.openai import OpenAIChatCompletionClient\n", + "from autogen_ext.tools.mcp import StdioServerParams, mcp_server_tools\n", + "\n", + "# Get the fetch tool from mcp-server-fetch.\n", + "fetch_mcp_server = StdioServerParams(command=\"uvx\", args=[\"mcp-server-fetch\"])\n", + "tools = await mcp_server_tools(fetch_mcp_server)\n", + "\n", + "# Create an agent that can use the fetch tool.\n", + "model_client = OpenAIChatCompletionClient(model=\"gpt-4o\")\n", + "agent = AssistantAgent(name=\"fetcher\", model_client=model_client, tools=tools, reflect_on_tool_use=True) # type: ignore\n", + "\n", + "# Let the agent fetch the content of a URL and summarize it.\n", + "result = await agent.run(task=\"Summarize the content of https://en.wikipedia.org/wiki/Seattle\")\n", + "assert isinstance(result.messages[-1], TextMessage)\n", + "print(result.messages[-1].content)\n", + "\n", + "# Close the connection to the model client.\n", + "await model_client.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Langchain Tools\n", + "\n", + "You can also use tools from the Langchain library\n", + "by wrapping them in {py:class}`~autogen_ext.tools.langchain.LangChainToolAdapter`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---------- assistant ----------\n", + "[FunctionCall(id='call_BEYRkf53nBS1G2uG60wHP0zf', arguments='{\"query\":\"df[\\'Age\\'].mean()\"}', name='python_repl_ast')]\n", + "[Prompt tokens: 111, Completion tokens: 22]\n", + "---------- assistant ----------\n", + "[FunctionExecutionResult(content='29.69911764705882', call_id='call_BEYRkf53nBS1G2uG60wHP0zf')]\n", + "---------- assistant ----------\n", + "29.69911764705882\n", + "---------- Summary ----------\n", + "Number of inner messages: 2\n", + "Total prompt tokens: 111\n", + "Total completion tokens: 22\n", + "Duration: 0.62 seconds\n" + ] }, - "nbformat": 4, - "nbformat_minor": 2 + { + "data": { + "text/plain": [ + "Response(chat_message=ToolCallSummaryMessage(source='assistant', models_usage=None, content='29.69911764705882', type='ToolCallSummaryMessage'), inner_messages=[ToolCallRequestEvent(source='assistant', models_usage=RequestUsage(prompt_tokens=111, completion_tokens=22), content=[FunctionCall(id='call_BEYRkf53nBS1G2uG60wHP0zf', arguments='{\"query\":\"df[\\'Age\\'].mean()\"}', name='python_repl_ast')], type='ToolCallRequestEvent'), ToolCallExecutionEvent(source='assistant', models_usage=None, content=[FunctionExecutionResult(content='29.69911764705882', call_id='call_BEYRkf53nBS1G2uG60wHP0zf')], type='ToolCallExecutionEvent')])" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "from autogen_ext.tools.langchain import LangChainToolAdapter\n", + "from langchain_experimental.tools.python.tool import PythonAstREPLTool\n", + "\n", + "df = pd.read_csv(\"https://raw.githubusercontent.com/pandas-dev/pandas/main/doc/data/titanic.csv\")\n", + "tool = LangChainToolAdapter(PythonAstREPLTool(locals={\"df\": df}))\n", + "model_client = OpenAIChatCompletionClient(model=\"gpt-4o\")\n", + "agent = AssistantAgent(\n", + " \"assistant\", tools=[tool], model_client=model_client, system_message=\"Use the `df` variable to access the dataset.\"\n", + ")\n", + "await Console(\n", + " agent.on_messages_stream(\n", + " [TextMessage(content=\"What's the average age of the passengers?\", source=\"user\")], CancellationToken()\n", + " ),\n", + " output_stats=True,\n", + ")\n", + "\n", + "await model_client.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Parallel Tool Calls\n", + "\n", + "Some models support parallel tool calls, which can be useful for tasks that require multiple tools to be called simultaneously.\n", + "By default, if the model client produces multiple tool calls, {py:class}`~autogen_agentchat.agents.AssistantAgent`\n", + "will call the tools in parallel.\n", + "\n", + "You may want to disable parallel tool calls when the tools have side effects that may interfere with each other, or,\n", + "when agent behavior needs to be consistent across different models.\n", + "This should be done at the model client level.\n", + "\n", + "For {py:class}`~autogen_ext.models.openai.OpenAIChatCompletionClient` and {py:class}`~autogen_ext.models.openai.AzureOpenAIChatCompletionClient`,\n", + "set `parallel_tool_calls=False` to disable parallel tool calls." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_client_no_parallel_tool_call = OpenAIChatCompletionClient(\n", + " model=\"gpt-4o\",\n", + " parallel_tool_calls=False, # type: ignore\n", + ")\n", + "agent_no_parallel_tool_call = AssistantAgent(\n", + " name=\"assistant\",\n", + " model_client=model_client_no_parallel_tool_call,\n", + " tools=[web_search],\n", + " system_message=\"Use tools to solve tasks.\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Running an Agent in a Loop\n", + "\n", + "The {py:class}`~autogen_agentchat.agents.AssistantAgent` executes one\n", + "step at a time: one model call, followed by one tool call (or parallel tool calls), and then\n", + "an optional reflection.\n", + "\n", + "To run it in a loop, for example, running it until it stops producing\n", + "tool calls, please refer to [Single-Agent Team](./teams.ipynb#single-agent-team)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Structured Output\n", + "\n", + "Structured output allows models to return structured JSON text with pre-defined schema\n", + "provided by the application. Different from JSON-mode, the schema can be provided\n", + "as a [Pydantic BaseModel](https://docs.pydantic.dev/latest/concepts/models/)\n", + "class, which can also be used to validate the output.\n", + "\n", + "Once you specify the base model class in the `output_content_type` parameter\n", + "of the {py:class}`~autogen_agentchat.agents.AssistantAgent` constructor,\n", + "the agent will respond with a {py:class}`~autogen_agentchat.messages.StructuredMessage`\n", + "whose `content`'s type is the type of the base model class.\n", + "\n", + "This way, you can integrate agent's response directly into your application\n", + "and use the model's output as a structured object.\n", + "\n", + "```{note}\n", + "When the `output_content_type` is set, it by default requires the agent to reflect on the tool use\n", + "and return the a structured output message based on the tool call result.\n", + "You can disable this behavior by setting `reflect_on_tool_use=False` explictly.\n", + "```\n", + "\n", + "Structured output is also useful for incorporating Chain-of-Thought\n", + "reasoning in the agent's responses.\n", + "See the example below for how to use structured output with the assistant agent." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---------- user ----------\n", + "I am happy.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---------- assistant ----------\n", + "{\n", + " \"thoughts\": \"The user explicitly states they are happy.\",\n", + " \"response\": \"happy\"\n", + "}\n", + "Thought: The user explicitly states they are happy.\n", + "Response: happy\n" + ] + } + ], + "source": [ + "from typing import Literal\n", + "\n", + "from pydantic import BaseModel\n", + "\n", + "\n", + "# The response format for the agent as a Pydantic base model.\n", + "class AgentResponse(BaseModel):\n", + " thoughts: str\n", + " response: Literal[\"happy\", \"sad\", \"neutral\"]\n", + "\n", + "\n", + "# Create an agent that uses the OpenAI GPT-4o model.\n", + "model_client = OpenAIChatCompletionClient(model=\"gpt-4o\")\n", + "agent = AssistantAgent(\n", + " \"assistant\",\n", + " model_client=model_client,\n", + " system_message=\"Categorize the input as happy, sad, or neutral following the JSON format.\",\n", + " # Define the output content type of the agent.\n", + " output_content_type=AgentResponse,\n", + ")\n", + "\n", + "result = await Console(agent.run_stream(task=\"I am happy.\"))\n", + "\n", + "# Check the last message in the result, validate its type, and print the thoughts and response.\n", + "assert isinstance(result.messages[-1], StructuredMessage)\n", + "assert isinstance(result.messages[-1].content, AgentResponse)\n", + "print(\"Thought: \", result.messages[-1].content.thoughts)\n", + "print(\"Response: \", result.messages[-1].content.response)\n", + "await model_client.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Streaming Tokens\n", + "\n", + "You can stream the tokens generated by the model client by setting `model_client_stream=True`.\n", + "This will cause the agent to yield {py:class}`~autogen_agentchat.messages.ModelClientStreamingChunkEvent` messages\n", + "in {py:meth}`~autogen_agentchat.agents.BaseChatAgent.on_messages_stream` and {py:meth}`~autogen_agentchat.agents.BaseChatAgent.run_stream`.\n", + "\n", + "The underlying model API must support streaming tokens for this to work.\n", + "Please check with your model provider to see if this is supported." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "source='assistant' models_usage=None content='Two' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' cities' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' in' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' South' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' America' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' are' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' Buenos' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' Aires' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' in' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' Argentina' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' and' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' São' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' Paulo' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' in' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' Brazil' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content='.' type='ModelClientStreamingChunkEvent'\n", + "Response(chat_message=TextMessage(source='assistant', models_usage=RequestUsage(prompt_tokens=0, completion_tokens=0), content='Two cities in South America are Buenos Aires in Argentina and São Paulo in Brazil.', type='TextMessage'), inner_messages=[])\n" + ] + } + ], + "source": [ + "model_client = OpenAIChatCompletionClient(model=\"gpt-4o\")\n", + "\n", + "streaming_assistant = AssistantAgent(\n", + " name=\"assistant\",\n", + " model_client=model_client,\n", + " system_message=\"You are a helpful assistant.\",\n", + " model_client_stream=True, # Enable streaming tokens.\n", + ")\n", + "\n", + "# Use an async function and asyncio.run() in a script.\n", + "async for message in streaming_assistant.on_messages_stream( # type: ignore\n", + " [TextMessage(content=\"Name two cities in South America\", source=\"user\")],\n", + " cancellation_token=CancellationToken(),\n", + "):\n", + " print(message)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can see the streaming chunks in the output above.\n", + "The chunks are generated by the model client and are yielded by the agent as they are received.\n", + "The final response, the concatenation of all the chunks, is yielded right after the last chunk.\n", + "\n", + "Similarly, {py:meth}`~autogen_agentchat.agents.BaseChatAgent.run_stream` will also yield the same streaming chunks,\n", + "followed by a full text message right after the last chunk." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "source='user' models_usage=None content='Name two cities in North America.' type='TextMessage'\n", + "source='assistant' models_usage=None content='Two' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' cities' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' in' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' North' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' America' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' are' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' New' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' York' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' City' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' in' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' the' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' United' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' States' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' and' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' Toronto' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' in' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content=' Canada' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=None content='.' type='ModelClientStreamingChunkEvent'\n", + "source='assistant' models_usage=RequestUsage(prompt_tokens=0, completion_tokens=0) content='Two cities in North America are New York City in the United States and Toronto in Canada.' type='TextMessage'\n", + "TaskResult(messages=[TextMessage(source='user', models_usage=None, content='Name two cities in North America.', type='TextMessage'), TextMessage(source='assistant', models_usage=RequestUsage(prompt_tokens=0, completion_tokens=0), content='Two cities in North America are New York City in the United States and Toronto in Canada.', type='TextMessage')], stop_reason=None)\n" + ] + } + ], + "source": [ + "async for message in streaming_assistant.run_stream(task=\"Name two cities in North America.\"): # type: ignore\n", + " print(message)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Model Context\n", + "\n", + "{py:class}`~autogen_agentchat.agents.AssistantAgent` has a `model_context`\n", + "parameter that can be used to pass in a {py:class}`~autogen_core.model_context.ChatCompletionContext`\n", + "object. This allows the agent to use different model contexts, such as\n", + "{py:class}`~autogen_core.model_context.BufferedChatCompletionContext` to\n", + "limit the context sent to the model.\n", + "\n", + "By default, {py:class}`~autogen_agentchat.agents.AssistantAgent` uses\n", + "the {py:class}`~autogen_core.model_context.UnboundedChatCompletionContext`\n", + "which sends the full conversation history to the model. To limit the context\n", + "to the last `n` messages, you can use the {py:class}`~autogen_core.model_context.BufferedChatCompletionContext`.\n", + "To limit the context by token count, you can use the\n", + "{py:class}`~autogen_core.model_context.TokenLimitedChatCompletionContext`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from autogen_core.model_context import BufferedChatCompletionContext\n", + "\n", + "# Create an agent that uses only the last 5 messages in the context to generate responses.\n", + "agent = AssistantAgent(\n", + " name=\"assistant\",\n", + " model_client=model_client,\n", + " tools=[web_search],\n", + " system_message=\"Use tools to solve tasks.\",\n", + " model_context=BufferedChatCompletionContext(buffer_size=5), # Only use the last 5 messages in the context.\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Other Preset Agents\n", + "\n", + "The following preset agents are available:\n", + "\n", + "- {py:class}`~autogen_agentchat.agents.UserProxyAgent`: An agent that takes user input returns it as responses.\n", + "- {py:class}`~autogen_agentchat.agents.CodeExecutorAgent`: An agent that can execute code.\n", + "- {py:class}`~autogen_ext.agents.openai.OpenAIAssistantAgent`: An agent that is backed by an OpenAI Assistant, with ability to use custom tools.\n", + "- {py:class}`~autogen_ext.agents.web_surfer.MultimodalWebSurfer`: A multi-modal agent that can search the web and visit web pages for information.\n", + "- {py:class}`~autogen_ext.agents.file_surfer.FileSurfer`: An agent that can search and browse local files for information.\n", + "- {py:class}`~autogen_ext.agents.video_surfer.VideoSurfer`: An agent that can watch videos for information." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Next Step\n", + "\n", + "Having explored the usage of the {py:class}`~autogen_agentchat.agents.AssistantAgent`, we can now proceed to the next section to learn about the teams feature in AgentChat.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 }