From 3aa19355c0e8cc57b53f5eee5daab8911138e367 Mon Sep 17 00:00:00 2001 From: Parker Stafford Date: Fri, 18 Oct 2024 10:39:43 -0700 Subject: [PATCH 01/12] get tool cals working for non tool roles --- app/src/components/trace/SpanKindIcon.tsx | 2 +- .../playground/PlaygroundChatTemplate.tsx | 65 ++++++++++++++-- app/src/pages/playground/PlaygroundOutput.tsx | 8 +- app/src/pages/playground/PlaygroundTool.tsx | 17 ++--- .../playground/PlaygroundToolCallsEditor.tsx | 75 +++++++++++++++++++ app/src/pages/playground/ToolOutline.tsx | 24 ++++++ app/src/pages/playground/ToolToggleButton.tsx | 41 ++++++++++ app/src/schemas/index.ts | 2 +- .../schemas/{toolSchema.ts => toolSchemas.ts} | 62 +++++++++++++-- app/src/store/playground/playgroundStore.tsx | 23 +++++- app/src/store/playground/types.ts | 32 ++------ 11 files changed, 299 insertions(+), 52 deletions(-) create mode 100644 app/src/pages/playground/PlaygroundToolCallsEditor.tsx create mode 100644 app/src/pages/playground/ToolOutline.tsx create mode 100644 app/src/pages/playground/ToolToggleButton.tsx rename app/src/schemas/{toolSchema.ts => toolSchemas.ts} (61%) diff --git a/app/src/components/trace/SpanKindIcon.tsx b/app/src/components/trace/SpanKindIcon.tsx index e7dce5bd89..3a16a8e045 100644 --- a/app/src/components/trace/SpanKindIcon.tsx +++ b/app/src/components/trace/SpanKindIcon.tsx @@ -36,7 +36,7 @@ const ToolSVG = () => ( ); -const ToolFilledSVG = () => ( +export const ToolFilledSVG = () => ( + {message.role !== "tool" && ( + { + updateInstance({ + instanceId: playgroundInstanceId, + patch: { + template: { + __type: "chat", + messages: template.messages.map((msg) => { + if (msg.id === message.id) { + return { + ...msg, + toolCalls: hasTools + ? undefined + : [createOpenAIToolCall()], + }; + } + return msg; + }), + }, + }, + }); + }} + /> + )} + {message.content != null && ( )} @@ -274,7 +315,7 @@ function SortableMessageItem({ } > -
+ -
+ {hasTools && message.role !== "tool" && ( + + + + + + + + )} + ); diff --git a/app/src/pages/playground/PlaygroundOutput.tsx b/app/src/pages/playground/PlaygroundOutput.tsx index 1291988cee..773e126379 100644 --- a/app/src/pages/playground/PlaygroundOutput.tsx +++ b/app/src/pages/playground/PlaygroundOutput.tsx @@ -9,7 +9,7 @@ import { useNotifyError } from "@phoenix/contexts"; import { useCredentialsContext } from "@phoenix/contexts/CredentialsContext"; import { usePlaygroundContext } from "@phoenix/contexts/PlaygroundContext"; import { useChatMessageStyles } from "@phoenix/hooks/useChatMessageStyles"; -import type { ToolCall } from "@phoenix/store"; +import { OpenAIToolCall } from "@phoenix/schemas"; import { ChatMessage, generateMessageId } from "@phoenix/store"; import { assertUnreachable } from "@phoenix/typeUtils"; @@ -52,7 +52,9 @@ function PlaygroundOutputMessage({ message }: { message: ChatMessage }) { > {toolCall.function.name}( {JSON.stringify( - JSON.parse(toolCall.function.arguments), + typeof toolCall.function.arguments === "string" + ? JSON.parse(toolCall.function.arguments) + : toolCall.function.arguments, null, 2 )} @@ -255,7 +257,7 @@ function PlaygroundOutputText(props: PlaygroundInstanceProps) { } const [output, setOutput] = useState(undefined); - const [toolCalls, setToolCalls] = useState([]); + const [toolCalls, setToolCalls] = useState([]); const azureModelParams = instance.model.provider === "AZURE_OPENAI" diff --git a/app/src/pages/playground/PlaygroundTool.tsx b/app/src/pages/playground/PlaygroundTool.tsx index d3e30bca2e..188aee563e 100644 --- a/app/src/pages/playground/PlaygroundTool.tsx +++ b/app/src/pages/playground/PlaygroundTool.tsx @@ -6,8 +6,8 @@ import { Button, Card, Flex, Icon, Icons } from "@arizeai/components"; import { CopyToClipboardButton } from "@phoenix/components"; import { JSONEditor } from "@phoenix/components/code"; import { usePlaygroundContext } from "@phoenix/contexts/PlaygroundContext"; -import { toolJSONSchema, toolSchema } from "@phoenix/schemas"; -import { Tool } from "@phoenix/store"; +import { openAIToolJSONSchema, openAIToolSchema } from "@phoenix/schemas"; +import { OpenAITool } from "@phoenix/store"; import { safelyParseJSON } from "@phoenix/utils/jsonUtils"; import { PlaygroundInstanceProps } from "./types"; @@ -16,7 +16,10 @@ export function PlaygroundTool({ playgroundInstanceId, tool, instanceTools, -}: PlaygroundInstanceProps & { tool: Tool; instanceTools: Tool[] }) { +}: PlaygroundInstanceProps & { + tool: OpenAITool; + instanceTools: OpenAITool[]; +}) { const updateInstance = usePlaygroundContext((state) => state.updateInstance); const [toolDefinition, setToolDefinition] = useState( @@ -34,7 +37,7 @@ export function PlaygroundTool({ // there is no "deepPassthrough" to allow for extra keys // at all levels of the schema, so we just use the json parsed value here, // knowing that it is valid with potentially extra keys - const { success } = toolSchema.safeParse(definition); + const { success } = openAIToolSchema.safeParse(definition); if (!success) { return; } @@ -45,10 +48,6 @@ export function PlaygroundTool({ t.id === tool.id ? { ...t, - // Don't use data here returned by safeParse, as we want to allow for extra keys, - // there is no "deepPassthrough" to allow for extra keys - // at all levels of the schema, so we just use the json parsed value here, - // knowing that it is valid with potentially extra keys definition, } : t @@ -92,7 +91,7 @@ export function PlaygroundTool({ ); diff --git a/app/src/pages/playground/PlaygroundToolCallsEditor.tsx b/app/src/pages/playground/PlaygroundToolCallsEditor.tsx new file mode 100644 index 0000000000..65cbf818e3 --- /dev/null +++ b/app/src/pages/playground/PlaygroundToolCallsEditor.tsx @@ -0,0 +1,75 @@ +import React, { useCallback, useState } from "react"; +import { JSONSchema7 } from "json-schema"; + +import { JSONEditor } from "@phoenix/components/code"; +import { usePlaygroundContext } from "@phoenix/contexts/PlaygroundContext"; +import { + openAIToolCallsJSONSchema, + openAIToolCallsSchema, +} from "@phoenix/schemas"; +import { ChatMessage } from "@phoenix/store"; +import { safelyParseJSON } from "@phoenix/utils/jsonUtils"; + +import { PlaygroundInstanceProps } from "./types"; + +/** + * Editor for message tool calls + */ +export function PlaygroundToolCallsEditor({ + playgroundInstanceId, + toolCalls, + templateMessages, + messageId, +}: PlaygroundInstanceProps & { + toolCalls: ChatMessage["toolCalls"]; + templateMessages: ChatMessage[]; + messageId: number; +}) { + const updateInstance = usePlaygroundContext((state) => state.updateInstance); + const [toolCallsValue, setToolCallsValue] = useState( + JSON.stringify(toolCalls, null, 2) + ); + + const onChange = useCallback( + (value: string) => { + setToolCallsValue(value); + const { json: definition } = safelyParseJSON(value); + if (definition == null) { + return; + } + // Don't use data here returned by safeParse, as we want to allow for extra keys, + // there is no "deepPassthrough" to allow for extra keys + // at all levels of the schema, so we just use the json parsed value here, + // knowing that it is valid with potentially extra keys + const { success } = openAIToolCallsSchema.safeParse(definition); + if (!success) { + return; + } + updateInstance({ + instanceId: playgroundInstanceId, + patch: { + template: { + __type: "chat", + messages: templateMessages.map((m) => + messageId === m.id + ? { + ...m, + toolCalls: definition, + } + : m + ), + }, + }, + }); + }, + [messageId, playgroundInstanceId, templateMessages, updateInstance] + ); + + return ( + + ); +} diff --git a/app/src/pages/playground/ToolOutline.tsx b/app/src/pages/playground/ToolOutline.tsx new file mode 100644 index 0000000000..19138fa37c --- /dev/null +++ b/app/src/pages/playground/ToolOutline.tsx @@ -0,0 +1,24 @@ +import React from "react"; + +export const ToolOutline = () => ( + + + + + + +); diff --git a/app/src/pages/playground/ToolToggleButton.tsx b/app/src/pages/playground/ToolToggleButton.tsx new file mode 100644 index 0000000000..561685bfde --- /dev/null +++ b/app/src/pages/playground/ToolToggleButton.tsx @@ -0,0 +1,41 @@ +import React from "react"; + +import { + Button, + Icon, + Icons, + Tooltip, + TooltipTrigger, +} from "@arizeai/components"; + +import { ToolOutline } from "./ToolOutline"; + +export function ToolToggleButton({ + hasTools, + onClick, +}: { + hasTools: boolean; + onClick: () => void; +}) { + return ( +
+ +
+ ); +} diff --git a/app/src/schemas/index.ts b/app/src/schemas/index.ts index c446331ef4..d54e007283 100644 --- a/app/src/schemas/index.ts +++ b/app/src/schemas/index.ts @@ -1 +1 @@ -export * from "./toolSchema"; +export * from "./toolSchemas"; diff --git a/app/src/schemas/toolSchema.ts b/app/src/schemas/toolSchemas.ts similarity index 61% rename from app/src/schemas/toolSchema.ts rename to app/src/schemas/toolSchemas.ts index 91388965ec..dbcd3ea93d 100644 --- a/app/src/schemas/toolSchema.ts +++ b/app/src/schemas/toolSchemas.ts @@ -2,13 +2,14 @@ import { z } from "zod"; import zodToJsonSchema from "zod-to-json-schema"; /** - * The schema for a tool definition + * The schema for an OpenAI tool definition * @see https://platform.openai.com/docs/guides/structured-outputs/supported-schemas * * Note: The nested passThrough's are used to allow for extra keys in JSON schema, however, they do not actually - * allow for extra keys when the zod schema is used for parsing. + * allow for extra keys when the zod schema is used for parsing. This is to allow more flexibility for users + * to define their own tools according */ -export const toolSchema = z +export const openAIToolSchema = z .object({ type: z.literal("function").describe("The type of the tool"), function: z @@ -77,11 +78,62 @@ export const toolSchema = z * The type of a tool definition * @see https://platform.openai.com/docs/guides/structured-outputs/supported-schemas */ -export type ToolDefinition = z.infer; +export type OpenAIToolDefinition = z.infer; /** * The JSON schema for a tool definition */ -export const toolJSONSchema = zodToJsonSchema(toolSchema, { +export const openAIToolJSONSchema = zodToJsonSchema(openAIToolSchema, { removeAdditionalStrategy: "passthrough", }); + +/** + * The schema for an OpenAI tool call, this is what a message that calls a tool looks like + * + * Note: The nested passThrough's are used to allow for extra keys in JSON schema, however, they do not actually + * allow for extra keys when the zod schema is used for parsing. This is to allow more flexibility for users + * to define their own tool calls according + */ +export const openAIToolCallSchema = z.object({ + id: z.string().describe("The ID of the tool call"), + function: z + .object({ + name: z.string().describe("The name of the function"), + arguments: z + .union([z.record(z.unknown()).optional(), z.string()]) + .describe("The arguments for the function"), + }) + .describe("The function that is being called") + .passthrough(), +}); + +/** + * The type of an OpenAI tool call + * + * @example + * ```typescript + * { + * id: "1", + * function: { + * name: "getCurrentWeather", + * arguments: "{ \"city\": \"San Francisco\" }" + * } + * } + * ``` + */ +export type OpenAIToolCall = z.infer; + +/** + * The zod schema for multiple OpenAI Tool Calls + */ +export const openAIToolCallsSchema = z.array(openAIToolCallSchema); + +/** + * The JSON schema for multiple OpenAI tool calls + */ +export const openAIToolCallsJSONSchema = zodToJsonSchema( + openAIToolCallsSchema, + { + removeAdditionalStrategy: "passthrough", + } +); diff --git a/app/src/store/playground/playgroundStore.tsx b/app/src/store/playground/playgroundStore.tsx index 47b72033d5..409b455a67 100644 --- a/app/src/store/playground/playgroundStore.tsx +++ b/app/src/store/playground/playgroundStore.tsx @@ -7,17 +7,18 @@ import { DEFAULT_CHAT_ROLE, DEFAULT_MODEL_PROVIDER, } from "@phoenix/constants/generativeConstants"; +import { OpenAIToolCall } from "@phoenix/schemas"; import { GenAIOperationType, InitialPlaygroundState, isManualInput, + OpenAITool, PlaygroundChatTemplate, PlaygroundInputMode, PlaygroundInstance, PlaygroundState, PlaygroundTextCompletionTemplate, - Tool, } from "./types"; let playgroundInstanceId = 0; @@ -106,7 +107,25 @@ export function createPlaygroundInstance(): PlaygroundInstance { }; } -export function createTool(toolNumber: number): Tool { +/** + * Creates an empty OpenAI tool call with fields but no values filled in + */ +export function createOpenAIToolCall(): OpenAIToolCall { + return { + id: "", + function: { + arguments: {}, + name: "", + }, + }; +} + +/** + * Creates a default tool with a unique ID and a function definition + * @param toolNumber the number of the tool in that instance for example instance.tools.length + 1 + * @returns a {@link Tool} with a unique ID and a function definition + */ +export function createOpenAITool(toolNumber: number): OpenAITool { return { id: generateToolId(), definition: { diff --git a/app/src/store/playground/types.ts b/app/src/store/playground/types.ts index b84eedca6e..4210941b6d 100644 --- a/app/src/store/playground/types.ts +++ b/app/src/store/playground/types.ts @@ -1,5 +1,5 @@ import { TemplateLanguage } from "@phoenix/components/templateEditor/types"; -import { ToolDefinition } from "@phoenix/schemas"; +import { OpenAIToolCall, OpenAIToolDefinition } from "@phoenix/schemas"; export type GenAIOperationType = "chat" | "text_completion"; /** @@ -8,27 +8,6 @@ export type GenAIOperationType = "chat" | "text_completion"; */ export type PlaygroundInputMode = "manual" | "dataset"; -/** - * A tool call that invokes a function with JSON arguments - * @example - * ```typescript - * { - * id: "1", - * function: { - * name: "getCurrentWeather", - * arguments: "{ \"city\": \"San Francisco\" }" - * } - * } - * ``` - */ -export type ToolCall = { - id: string; - function: { - name: string; - arguments: string; - }; -}; - /** * A chat message with a role and content * @example { role: "user", content: "What is the weather in San Francisco?" } @@ -52,7 +31,8 @@ export type ChatMessage = { id: number; role: ChatMessageRole; content?: string; - toolCalls?: ToolCall[]; + toolCalls?: OpenAIToolCall[]; + toolCallId?: string; }; /** @@ -101,9 +81,9 @@ export type ModelConfig = { /** * The type of a tool in the playground */ -export type Tool = { +export type OpenAITool = { id: number; - definition: Partial; + definition: Partial; }; /** @@ -119,7 +99,7 @@ export interface PlaygroundInstance { */ id: number; template: PlaygroundTemplate; - tools: Tool[]; + tools: OpenAITool[]; /** * How the LLM should choose the tool to use * @default "auto" From e6e7e226cf4201026fdab8b0c27d7442f9f7635f Mon Sep 17 00:00:00 2001 From: Parker Stafford Date: Fri, 18 Oct 2024 10:52:26 -0700 Subject: [PATCH 02/12] renaming --- ...tor.tsx => ChatMessageToolCallsEditor.tsx} | 2 +- .../playground/PlaygroundChatTemplate.tsx | 4 +-- app/src/pages/playground/ToolToggleButton.tsx | 36 +++++++++---------- 3 files changed, 20 insertions(+), 22 deletions(-) rename app/src/pages/playground/{PlaygroundToolCallsEditor.tsx => ChatMessageToolCallsEditor.tsx} (97%) diff --git a/app/src/pages/playground/PlaygroundToolCallsEditor.tsx b/app/src/pages/playground/ChatMessageToolCallsEditor.tsx similarity index 97% rename from app/src/pages/playground/PlaygroundToolCallsEditor.tsx rename to app/src/pages/playground/ChatMessageToolCallsEditor.tsx index 65cbf818e3..1b4adaeb6a 100644 --- a/app/src/pages/playground/PlaygroundToolCallsEditor.tsx +++ b/app/src/pages/playground/ChatMessageToolCallsEditor.tsx @@ -15,7 +15,7 @@ import { PlaygroundInstanceProps } from "./types"; /** * Editor for message tool calls */ -export function PlaygroundToolCallsEditor({ +export function ChatMessageToolCallsEditor({ playgroundInstanceId, toolCalls, templateMessages, diff --git a/app/src/pages/playground/PlaygroundChatTemplate.tsx b/app/src/pages/playground/PlaygroundChatTemplate.tsx index 3d305e143e..c463f4d62b 100644 --- a/app/src/pages/playground/PlaygroundChatTemplate.tsx +++ b/app/src/pages/playground/PlaygroundChatTemplate.tsx @@ -40,8 +40,8 @@ import { PlaygroundChatTemplate as PlaygroundChatTemplateType, } from "@phoenix/store"; +import { ChatMessageToolCallsEditor } from "./ChatMessageToolCallsEditor"; import { MessageRolePicker } from "./MessageRolePicker"; -import { PlaygroundToolCallsEditor } from "./PlaygroundToolCallsEditor"; import { PlaygroundTools } from "./PlaygroundTools"; import { ToolToggleButton } from "./ToolToggleButton"; import { PlaygroundInstanceProps } from "./types"; @@ -339,7 +339,7 @@ function SortableMessageItem({ - void; }) { return ( -
- -
+ +