OpenAI Responses API Plugin

livekit-agents/livekit/agents/llm/_provider_format/__init__.py

@@ -1,3 +1,3 @@
 from . import anthropic, aws, google, mistralai, openai
 
-__all__ = ["openai", "google", "aws", "anthropic", "mistralai"]
+__all__ = ["openai", "openai.responses", "google", "aws", "anthropic", "mistralai"]

livekit-agents/livekit/agents/llm/_provider_format/openai.py

@@ -115,6 +115,79 @@ def _to_image_content(image: llm.ImageContent) -> dict[str, Any]:
     }
 
 
+def to_responses_chat_ctx(
+    chat_ctx: llm.ChatContext, *, inject_dummy_user_message: bool = True
+) -> tuple[list[dict], Literal[None]]:
+    item_groups = group_tool_calls(chat_ctx)
+    items = []
+    for group in item_groups:
+        if not group.message and not group.tool_calls and not group.tool_outputs:
+            continue
+
+        if group.message:
+            msg = _to_responses_chat_item(group.message)
+            items.append(msg)
+
+        for tool_call in group.tool_calls:
+            call = {
+                "call_id": tool_call.call_id,
+                "type": "function_call",
+                "name": tool_call.name,
+                "arguments": tool_call.arguments,
+            }
+            items.append(call)
+
+        for tool_output in group.tool_outputs:
+            items.append(_to_responses_chat_item(tool_output))
+
+    return items, None
+
+
+def _to_responses_chat_item(msg: llm.ChatItem) -> dict[str, Any]:
+    if msg.type == "message":
+        list_content: list[dict[str, Any]] = []
+        text_content = ""
+        for content in msg.content:
+            if isinstance(content, str):
+                if text_content:
+                    text_content += "\n"
+                text_content += content
+            elif isinstance(content, llm.ImageContent):
+                list_content.append(_to_image_content(content))
+
+        if not list_content:
+            return {"role": msg.role, "content": text_content}
+
+        if text_content:
+            list_content.append({"type": "text", "text": text_content})
+
+        return {"role": msg.role, "content": list_content}
+
+    elif msg.type == "function_call":
+        return {
+            "role": "assistant",
+            "tool_calls": [
+                {
+                    "id": msg.call_id,
+                    "type": "function",
+                    "function": {
+                        "name": msg.name,
+                        "arguments": msg.arguments,
+                    },
+                }
+            ],
+        }
+
+    elif msg.type == "function_call_output":
+        return {
+            "type": "function_call_output",
+            "call_id": msg.call_id,
+            "output": msg.output,
+        }
+
+    raise ValueError(f"unsupported message type: {msg.type}")
+
+
 def to_fnc_ctx(tool_ctx: llm.ToolContext, *, strict: bool = True) -> list[dict[str, Any]]:
     schemas: list[dict[str, Any]] = []
     for tool in tool_ctx.function_tools.values():
@@ -135,3 +208,21 @@ def to_fnc_ctx(tool_ctx: llm.ToolContext, *, strict: bool = True) -> list[dict[s
             schemas.append(schema)
 
     return schemas
+
+
+def to_responses_fnc_ctx(tool_ctx: llm.ToolContext, *, strict: bool = True) -> list[dict[str, Any]]:
+    from livekit.plugins import openai
+
+    schemas: list[dict[str, Any]] = []
+    for tool in tool_ctx.flatten():
+        if isinstance(tool, llm.RawFunctionTool):
+            schema = tool.info.raw_schema
+            schema["type"] = "function"
+            schemas.append(schema)
+        elif isinstance(tool, llm.FunctionTool):
+            schema = llm.utils.build_legacy_openai_schema(tool, internally_tagged=True)
+            schemas.append(schema)
+        elif isinstance(tool, openai.tools.OpenAITool):
+            schemas.append(tool.to_dict())
+
+    return schemas

livekit-agents/livekit/agents/llm/chat_context.py

@@ -442,7 +442,10 @@ def to_dict(
 
     @overload
     def to_provider_format(
-        self, format: Literal["openai"], *, inject_dummy_user_message: bool = True
+        self,
+        format: Literal["openai", "openai.responses"],
+        *,
+        inject_dummy_user_message: bool = True,
     ) -> tuple[list[dict], Literal[None]]: ...
 
     @overload
@@ -470,7 +473,8 @@ def to_provider_format(self, format: str, **kwargs: Any) -> tuple[list[dict], An
 
     def to_provider_format(
         self,
-        format: Literal["openai", "google", "aws", "anthropic", "mistralai"] | str,
+        format: Literal["openai", "openai.responses", "google", "aws", "anthropic", "mistralai"]
+        | str,
         *,
         inject_dummy_user_message: bool = True,
         **kwargs: Any,
@@ -487,6 +491,8 @@ def to_provider_format(
 
         if format == "openai":
             return _provider_format.openai.to_chat_ctx(self, **kwargs)
+        elif format == "openai.responses":
+            return _provider_format.openai.to_responses_chat_ctx(self, **kwargs)
         elif format == "google":
             return _provider_format.google.to_chat_ctx(self, **kwargs)
         elif format == "aws":

livekit-agents/livekit/agents/llm/tool_context.py

@@ -381,7 +381,7 @@ def copy(self) -> ToolContext:
 
     @overload
     def parse_function_tools(
-        self, format: Literal["openai"], *, strict: bool = True
+        self, format: Literal["openai", "openai.responses"], *, strict: bool = True
     ) -> list[dict[str, Any]]: ...
 
     @overload
@@ -406,6 +406,8 @@ def parse_function_tools(
         """Parse the function tools to a provider-specific schema."""
         if format == "openai":
             return _provider_format.openai.to_fnc_ctx(self, **kwargs)
+        elif format == "openai.responses":
+            return _provider_format.openai.to_responses_fnc_ctx(self, **kwargs)
         elif format == "google":
             return _provider_format.google.to_fnc_ctx(self, **kwargs)
         elif format == "anthropic":

livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/__init__.py

@@ -23,7 +23,7 @@
 https://docs.livekit.io/agents/integrations/llm/ for more information.
 """
 
-from . import realtime
+from . import realtime, responses, tools
 from .embeddings import EmbeddingData, create_embeddings
 from .llm import LLM, LLMStream
 from .models import (
@@ -50,6 +50,8 @@
     "create_embeddings",
     "EmbeddingData",
     "realtime",
+    "tools",
+    "responses",
     "__version__",
 ]
 

livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/responses/__init__.py

@@ -0,0 +1,3 @@
+from .llm import LLM
+
+__all__ = ["LLM"]