Adding extra content to OpenAI LLM. Improving function call grouping.

livekit-agents/livekit/agents/inference/llm.py

@@ -273,6 +273,7 @@ async def _run(self) -> None:
         self._tool_call_id: str | None = None
         self._fnc_name: str | None = None
         self._fnc_raw_arguments: str | None = None
+        self._tool_extra: dict[str, Any] | None = None
         self._tool_index: int | None = None
         retryable = True
 
@@ -375,17 +376,21 @@ def _parse_choice(
                                     arguments=self._fnc_raw_arguments or "",
                                     name=self._fnc_name or "",
                                     call_id=self._tool_call_id or "",
+                                    extra=self._tool_extra,
                                 )
                             ],
                         ),
                     )
                     self._tool_call_id = self._fnc_name = self._fnc_raw_arguments = None
+                    self._tool_extra = None
 
                 if tool.function.name:
                     self._tool_index = tool.index
                     self._tool_call_id = tool.id
                     self._fnc_name = tool.function.name
                     self._fnc_raw_arguments = tool.function.arguments or ""
+                    # Extract extra from tool call (e.g., Google thought signatures)
+                    self._tool_extra = getattr(tool, "extra_content", None)
                 elif tool.function.arguments:
                     self._fnc_raw_arguments += tool.function.arguments  # type: ignore
 
@@ -403,21 +408,30 @@ def _parse_choice(
                             arguments=self._fnc_raw_arguments or "",
                             name=self._fnc_name or "",
                             call_id=self._tool_call_id or "",
+                            extra=self._tool_extra,
                         )
                     ],
                 ),
             )
             self._tool_call_id = self._fnc_name = self._fnc_raw_arguments = None
+            self._tool_extra = None
             return call_chunk
 
         delta.content = llm_utils.strip_thinking_tokens(delta.content, thinking)
 
-        if not delta.content:
+        # Extract extra from delta (e.g., Google thought signatures on text parts)
+        delta_extra = getattr(delta, "extra_content", None)
+
+        if not delta.content and not delta_extra:
             return None
 
         return llm.ChatChunk(
             id=id,
-            delta=llm.ChoiceDelta(content=delta.content, role="assistant"),
+            delta=llm.ChoiceDelta(
+                content=delta.content,
+                role="assistant",
+                extra=delta_extra,
+            ),
         )
 
 

livekit-agents/livekit/agents/llm/_provider_format/openai.py

@@ -19,14 +19,17 @@ def to_chat_ctx(
 
         # one message can contain zero or more tool calls
         msg = _to_chat_item(group.message) if group.message else {"role": "assistant"}
-        tool_calls = [
-            {
+        tool_calls = []
+        for tool_call in group.tool_calls:
+            tc: dict[str, Any] = {
                 "id": tool_call.call_id,
                 "type": "function",
                 "function": {"name": tool_call.name, "arguments": tool_call.arguments},
             }
-            for tool_call in group.tool_calls
-        ]
+            # Include provider-specific extra content (e.g., Google thought signatures)
+            if tool_call.extra.get("google"):
+                tc["extra_content"] = {"google": tool_call.extra["google"]}
+            tool_calls.append(tc)
         if tool_calls:
             msg["tool_calls"] = tool_calls
         messages.append(msg)
@@ -53,26 +56,32 @@ def _to_chat_item(msg: llm.ChatItem) -> dict[str, Any]:
         if not list_content:
             # certain providers require text-only content in a string vs a list.
             # for max-compatibility, we will combine all text content into a single string.
-            return {"role": msg.role, "content": text_content}
+            result: dict[str, Any] = {"role": msg.role, "content": text_content}
+        else:
+            if text_content:
+                list_content.append({"type": "text", "text": text_content})
+            result = {"role": msg.role, "content": list_content}
 
-        if text_content:
-            list_content.append({"type": "text", "text": text_content})
-
-        return {"role": msg.role, "content": list_content}
+        # Include provider-specific extra content (e.g., Google thought signatures)
+        if msg.extra.get("google"):
+            result["extra_content"] = {"google": msg.extra["google"]}
+        return result
 
     elif msg.type == "function_call":
+        tc: dict[str, Any] = {
+            "id": msg.call_id,
+            "type": "function",
+            "function": {
+                "name": msg.name,
+                "arguments": msg.arguments,
+            },
+        }
+        # Include provider-specific extra content (e.g., Google thought signatures)
+        if msg.extra.get("google"):
+            tc["extra_content"] = {"google": msg.extra["google"]}
         return {
             "role": "assistant",
-            "tool_calls": [
-                {
-                    "id": msg.call_id,
-                    "type": "function",
-                    "function": {
-                        "name": msg.name,
-                        "arguments": msg.arguments,
-                    },
-                }
-            ],
+            "tool_calls": [tc],
         }
 
     elif msg.type == "function_call_output":

livekit-agents/livekit/agents/llm/_provider_format/utils.py

@@ -29,7 +29,12 @@ def group_tool_calls(chat_ctx: llm.ChatContext) -> list[_ChatItemGroup]:
     for item in chat_ctx.items:
         if (item.type == "message" and item.role == "assistant") or item.type == "function_call":
             # only assistant messages and function calls can be grouped
-            group_id = item.id.split("/")[0]
+            # For function calls, use group_id if available (for parallel function calls),
+            # otherwise fall back to id-based grouping for backwards compatibility
+            if item.type == "function_call" and item.group_id:
+                group_id = item.group_id
+            else:
+                group_id = item.id.split("/")[0]
             if group_id not in item_groups:
                 item_groups[group_id] = _ChatItemGroup().add(item)
             else:

livekit-agents/livekit/agents/llm/chat_context.py

@@ -183,6 +183,13 @@ class FunctionCall(BaseModel):
     arguments: str
     name: str
     created_at: float = Field(default_factory=time.time)
+    extra: dict[str, Any] = Field(default_factory=dict)
+    """Extra data for this function call. Can include provider-specific data
+    (e.g., extra["google"] for thought signatures)."""
+    group_id: str | None = None
+    """Optional group ID for parallel function calls. When multiple function calls
+    should be grouped together (e.g., parallel tool calls from a single API response),
+    set this to a shared value. If not set, falls back to using id for grouping."""
 
 
 class FunctionCallOutput(BaseModel):

livekit-agents/livekit/agents/llm/llm.py

@@ -52,12 +52,16 @@ class FunctionToolCall(BaseModel):
     name: str
     arguments: str
     call_id: str
+    extra: dict[str, Any] | None = None
+    """Provider-specific extra data (e.g., Google thought signatures)."""
 
 
 class ChoiceDelta(BaseModel):
     role: ChatRole | None = None
     content: str | None = None
     tool_calls: list[FunctionToolCall] = Field(default_factory=list)
+    extra: dict[str, Any] | None = None
+    """Provider-specific extra data (e.g., Google thought signatures)."""
 
 
 class ChatChunk(BaseModel):

livekit-agents/livekit/agents/voice/generation.py

@@ -150,6 +150,7 @@ async def _llm_inference_task(
                             call_id=tool.call_id,
                             name=tool.name,
                             arguments=tool.arguments,
+                            extra=tool.extra or {},
                         )
                         data.generated_functions.append(fnc_call)
                         function_ch.send_nowait(fnc_call)