From bf064576efcd9ad03d622baac58c39170baf5a8a Mon Sep 17 00:00:00 2001 From: Tony Powell Date: Mon, 28 Oct 2024 10:47:52 -0400 Subject: [PATCH 1/2] feat: convert tool call schemas between providers --- app/src/pages/playground/toolCallSchemas.ts | 144 ++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 app/src/pages/playground/toolCallSchemas.ts diff --git a/app/src/pages/playground/toolCallSchemas.ts b/app/src/pages/playground/toolCallSchemas.ts new file mode 100644 index 0000000000..5a97cc3ff7 --- /dev/null +++ b/app/src/pages/playground/toolCallSchemas.ts @@ -0,0 +1,144 @@ +import { z } from "zod"; + +import { assertUnreachable } from "@phoenix/typeUtils"; + +export const jsonSchema = z + .object({ + type: z.string(), + // content changes based on the type + // see https://json-schema.org/understanding-json-schema/reference/type + }) + .passthrough(); + +/** + * -------------------------------- + * Provider Schemas + * -------------------------------- + */ + +/** + * OpenAI tool call format + */ +export const openAIToolCallSchema = z.object({ + id: z.string().nullish(), + type: z.literal("function"), + function: z.object({ + name: z.string(), + description: z.string().nullish(), + parameters: jsonSchema, + }), +}); + +export type OpenAIToolCall = z.infer; + +/** + * Anthropic tool call format + */ +export const anthropicToolCallSchema = z.object({ + name: z.string(), + description: z.string(), + input_schema: jsonSchema, +}); + +export type AnthropicToolCall = z.infer; + +/** + * -------------------------------- + * Conversion Schemas + * -------------------------------- + */ + +/** + * Parse incoming object as an Anthropic tool call and immediately convert to OpenAI format + */ +export const anthropicToOpenAI = anthropicToolCallSchema.transform( + (anthropic): OpenAIToolCall => ({ + id: null, // generate? + type: "function", + function: { + name: anthropic.name, + description: anthropic.description, + parameters: anthropic.input_schema, + }, + }) +); + +/** + * Parse incoming object as an OpenAI tool call and immediately convert to Anthropic format + */ +export const openAIToAnthropic = openAIToolCallSchema.transform( + (openai): AnthropicToolCall => ({ + name: openai.function.name, + description: openai.function.description ?? openai.function.name, + input_schema: openai.function.parameters, + }) +); + +/** + * -------------------------------- + * Conversion Helpers + * -------------------------------- + */ + +// Helper type for provider identification +export const providerType = z.enum(["openai", "anthropic"]); +export type ProviderType = z.infer; + +/** + * Union of all tool call formats + * + * This is useful for functions that need to accept any tool call format + */ +export const anyToolCallSchema = z.union([ + openAIToolCallSchema, + anthropicToolCallSchema, +]); + +export type AnyToolCall = z.infer; + +/** + * Convert from any tool call format to OpenAI format + */ +export const toOpenAIFormat = ( + toolCall: AnyToolCall, + provider: ProviderType +): OpenAIToolCall => { + switch (provider) { + case "openai": + return toolCall as OpenAIToolCall; + case "anthropic": + return anthropicToOpenAI.parse(toolCall as AnthropicToolCall); + default: + assertUnreachable(provider); + } +}; + +/** + * Convert from OpenAI tool call format to any other format + */ +export const fromOpenAIFormat = ( + toolCall: OpenAIToolCall, + targetProvider: ProviderType +): AnyToolCall => { + switch (targetProvider) { + case "openai": + return toolCall; + case "anthropic": + return openAIToAnthropic.parse(toolCall); + default: + assertUnreachable(targetProvider); + } +}; + +/** + * Detect the provider of a tool call object + */ +export const detectProvider = (toolCall: unknown): ProviderType => { + if (openAIToolCallSchema.safeParse(toolCall).success) { + return "openai"; + } + if (anthropicToolCallSchema.safeParse(toolCall).success) { + return "anthropic"; + } + throw new Error("Unknown tool call format"); +}; From ac717a9560c8462945bf98c5e270b0b6052ebd7a Mon Sep 17 00:00:00 2001 From: Tony Powell Date: Mon, 28 Oct 2024 15:54:35 -0400 Subject: [PATCH 2/2] Split tool definition schemas from tool call schemas --- .../playground/ChatMessageToolCallsEditor.tsx | 7 +- app/src/pages/playground/PlaygroundOutput.tsx | 2 +- app/src/pages/playground/PlaygroundTool.tsx | 9 +- app/src/pages/playground/schemas.ts | 5 +- app/src/schemas/index.ts | 1 + app/src/schemas/toolCallSchemas.ts | 189 ++++++++++++++ app/src/schemas/toolSchemas.ts | 244 ++++++++++++------ app/src/store/playground/playgroundStore.tsx | 2 +- app/src/store/playground/types.ts | 4 +- 9 files changed, 368 insertions(+), 95 deletions(-) create mode 100644 app/src/schemas/toolCallSchemas.ts diff --git a/app/src/pages/playground/ChatMessageToolCallsEditor.tsx b/app/src/pages/playground/ChatMessageToolCallsEditor.tsx index 0415401e6c..66f1d606f4 100644 --- a/app/src/pages/playground/ChatMessageToolCallsEditor.tsx +++ b/app/src/pages/playground/ChatMessageToolCallsEditor.tsx @@ -3,15 +3,14 @@ import { JSONSchema7 } from "json-schema"; import { JSONEditor } from "@phoenix/components/code"; import { usePlaygroundContext } from "@phoenix/contexts/PlaygroundContext"; -import { - openAIToolCallsJSONSchema, - openAIToolCallsSchema, -} from "@phoenix/schemas"; +import { toolCallSchemas } from "@phoenix/schemas"; import { ChatMessage } from "@phoenix/store"; import { safelyParseJSON } from "@phoenix/utils/jsonUtils"; import { PlaygroundInstanceProps } from "./types"; +const { openAIToolCallsSchema, openAIToolCallsJSONSchema } = toolCallSchemas; + /** * Editor for message tool calls */ diff --git a/app/src/pages/playground/PlaygroundOutput.tsx b/app/src/pages/playground/PlaygroundOutput.tsx index 60c42612d1..679b074b53 100644 --- a/app/src/pages/playground/PlaygroundOutput.tsx +++ b/app/src/pages/playground/PlaygroundOutput.tsx @@ -9,7 +9,7 @@ import { useNotifyError } from "@phoenix/contexts"; import { useCredentialsContext } from "@phoenix/contexts/CredentialsContext"; import { usePlaygroundContext } from "@phoenix/contexts/PlaygroundContext"; import { useChatMessageStyles } from "@phoenix/hooks/useChatMessageStyles"; -import { OpenAIToolCall } from "@phoenix/schemas"; +import { OpenAIToolCall } from "@phoenix/schemas/toolCallSchemas"; import { ChatMessage, generateMessageId } from "@phoenix/store"; import { assertUnreachable } from "@phoenix/typeUtils"; diff --git a/app/src/pages/playground/PlaygroundTool.tsx b/app/src/pages/playground/PlaygroundTool.tsx index 3db1853939..39631db0b6 100644 --- a/app/src/pages/playground/PlaygroundTool.tsx +++ b/app/src/pages/playground/PlaygroundTool.tsx @@ -8,7 +8,10 @@ import { JSONEditor } from "@phoenix/components/code"; import { LazyEditorWrapper } from "@phoenix/components/code/LazyEditorWrapper"; import { SpanKindIcon } from "@phoenix/components/trace"; import { usePlaygroundContext } from "@phoenix/contexts/PlaygroundContext"; -import { openAIToolJSONSchema, openAIToolSchema } from "@phoenix/schemas"; +import { + openAIToolDefinitionJSONSchema, + openAIToolDefinitionSchema, +} from "@phoenix/schemas"; import { OpenAITool } from "@phoenix/store"; import { safelyParseJSON } from "@phoenix/utils/jsonUtils"; @@ -45,7 +48,7 @@ export function PlaygroundTool({ // there is no "deepPassthrough" to allow for extra keys // at all levels of the schema, so we just use the json parsed value here, // knowing that it is valid with potentially extra keys - const { success } = openAIToolSchema.safeParse(definition); + const { success } = openAIToolDefinitionSchema.safeParse(definition); if (!success) { return; } @@ -106,7 +109,7 @@ export function PlaygroundTool({ diff --git a/app/src/pages/playground/schemas.ts b/app/src/pages/playground/schemas.ts index 9dc568945e..fa238c07e6 100644 --- a/app/src/pages/playground/schemas.ts +++ b/app/src/pages/playground/schemas.ts @@ -7,7 +7,8 @@ import { ToolAttributePostfixes, } from "@arizeai/openinference-semantic-conventions"; -import { openAIToolCallSchema, openAIToolSchema } from "@phoenix/schemas"; +import { openAIToolDefinitionSchema } from "@phoenix/schemas"; +import { openAIToolCallSchema } from "@phoenix/schemas/toolCallSchemas"; import { ChatMessage } from "@phoenix/store"; import { isObject, Mutable, schemaForType } from "@phoenix/typeUtils"; import { safelyParseJSON } from "@phoenix/utils/jsonUtils"; @@ -220,7 +221,7 @@ export const toolJSONSchemaSchema = z }) // TODO(parker / apowell) - adjust this transformation with anthropic tool support https://github.com/Arize-ai/phoenix/issues/5100 .transform((o, ctx) => { - const { data, success } = openAIToolSchema.safeParse(o); + const { data, success } = openAIToolDefinitionSchema.safeParse(o); if (!success) { ctx.addIssue({ diff --git a/app/src/schemas/index.ts b/app/src/schemas/index.ts index d54e007283..62e525079a 100644 --- a/app/src/schemas/index.ts +++ b/app/src/schemas/index.ts @@ -1 +1,2 @@ export * from "./toolSchemas"; +export * as toolCallSchemas from "./toolCallSchemas"; diff --git a/app/src/schemas/toolCallSchemas.ts b/app/src/schemas/toolCallSchemas.ts new file mode 100644 index 0000000000..2924c4f6d0 --- /dev/null +++ b/app/src/schemas/toolCallSchemas.ts @@ -0,0 +1,189 @@ +import { z } from "zod"; +import zodToJsonSchema from "zod-to-json-schema"; + +import { assertUnreachable } from "@phoenix/typeUtils"; + +/** + * The schema for an OpenAI tool call, this is what a message that calls a tool looks like + * + * Note: The nested passThrough's are used to allow for extra keys in JSON schema, however, they do not actually + * allow for extra keys when the zod schema is used for parsing. This is to allow more flexibility for users + * to define their own tool calls according + */ +export const openAIToolCallSchema = z.object({ + id: z.string().describe("The ID of the tool call"), + function: z + .object({ + name: z.string().describe("The name of the function"), + arguments: z + .union([z.record(z.unknown()).optional(), z.string()]) + .describe("The arguments for the function"), + }) + .describe("The function that is being called") + .passthrough(), +}); + +/** + * The type of an OpenAI tool call + * + * @example + * ```typescript + * { + * id: "1", + * function: { + * name: "getCurrentWeather", + * arguments: "{ \"city\": \"San Francisco\" }" + * } + * } + * ``` + */ +export type OpenAIToolCall = z.infer; + +/** + * The zod schema for multiple OpenAI Tool Calls + */ +export const openAIToolCallsSchema = z.array(openAIToolCallSchema); + +/** + * The JSON schema for multiple OpenAI tool calls + */ +export const openAIToolCallsJSONSchema = zodToJsonSchema( + openAIToolCallsSchema, + { + removeAdditionalStrategy: "passthrough", + } +); + +/** + * The schema for an Anthropic tool call, this is what a message that calls a tool looks like + */ +export const anthropicToolCallSchema = z + .object({ + id: z.string().describe("The ID of the tool call"), + type: z.literal("tool_use"), + name: z.string().describe("The name of the tool"), + input: z.record(z.unknown()).describe("The input for the tool"), + }) + .passthrough(); + +/** + * The type of an Anthropic tool call + */ +export type AnthropicToolCall = z.infer; + +/** + * The zod schema for multiple Anthropic tool calls + */ +export const anthropicToolCallsSchema = z.array(anthropicToolCallSchema); + +/** + * The JSON schema for multiple Anthropic tool calls + */ +export const anthropicToolCallsJSONSchema = zodToJsonSchema( + anthropicToolCallsSchema, + { + removeAdditionalStrategy: "passthrough", + } +); + +/** + * -------------------------------- + * Conversion Schemas + * -------------------------------- + */ + +/** + * Parse incoming object as an Anthropic tool call and immediately convert to OpenAI format + */ +export const anthropicToOpenAI = anthropicToolCallSchema.transform( + (anthropic): OpenAIToolCall => ({ + id: anthropic.id, + function: { + name: anthropic.name, + arguments: anthropic.input, + }, + }) +); + +/** + * Parse incoming object as an OpenAI tool call and immediately convert to Anthropic format + */ +export const openAIToAnthropic = openAIToolCallSchema.transform( + (openai): AnthropicToolCall => ({ + id: openai.id, + type: "tool_use", + name: openai.function.name, + // REVIEW: anthropic wants a record always, openai wants string, record, or undefined + // whats the best way to handle this? + input: + typeof openai.function.arguments === "string" + ? { [openai.function.arguments]: openai.function.arguments } + : openai.function.arguments ?? {}, + }) +); + +/** + * -------------------------------- + * Conversion Helpers + * -------------------------------- + */ + +/** + * Union of all tool call formats + * + * This is useful for functions that need to accept any tool call format + */ +export const anyToolCallSchema = z.union([ + openAIToolCallSchema, + anthropicToolCallSchema, +]); + +export type AnyToolCall = z.infer; + +/** + * Detect the provider of a tool call object + */ +export const detectProvider = ( + toolCall: unknown +): { provider: ModelProvider; validatedToolCall: AnyToolCall } => { + let parsedToolCall: z.SafeParseReturnType; + + parsedToolCall = openAIToolCallSchema.safeParse(toolCall); + if (parsedToolCall.success) { + // we cannot disambiguate between azure openai and openai here + return { provider: "OPENAI", validatedToolCall: parsedToolCall.data }; + } + parsedToolCall = anthropicToolCallSchema.safeParse(toolCall); + if (parsedToolCall.success) { + return { provider: "ANTHROPIC", validatedToolCall: parsedToolCall.data }; + } + throw new Error("Unknown tool call format"); +}; + +export const toOpenAIFormat = (toolCall: AnyToolCall): OpenAIToolCall => { + const { provider, validatedToolCall } = detectProvider(toolCall); + switch (provider) { + case "AZURE_OPENAI": + case "OPENAI": + return validatedToolCall as OpenAIToolCall; + case "ANTHROPIC": + return anthropicToOpenAI.parse(validatedToolCall); + default: + assertUnreachable(provider); + } +}; + +export const fromOpenAIFormat = ( + toolCall: OpenAIToolCall, + targetProvider: ModelProvider +): AnyToolCall => { + switch (targetProvider) { + case "AZURE_OPENAI": + case "OPENAI": + return toolCall; + case "ANTHROPIC": + return openAIToAnthropic.parse(toolCall); + default: + assertUnreachable(targetProvider); + } +}; diff --git a/app/src/schemas/toolSchemas.ts b/app/src/schemas/toolSchemas.ts index dbcd3ea93d..8a2f3fd6f9 100644 --- a/app/src/schemas/toolSchemas.ts +++ b/app/src/schemas/toolSchemas.ts @@ -1,6 +1,48 @@ import { z } from "zod"; import zodToJsonSchema from "zod-to-json-schema"; +import { assertUnreachable } from "@phoenix/typeUtils"; + +const jsonSchemaZodSchema = z + .object({ + type: z.literal("object"), + properties: z + .record( + z + .object({ + type: z + .enum([ + "string", + "number", + "boolean", + "object", + "array", + "null", + "integer", + ]) + .describe("The type of the parameter"), + description: z + .string() + .optional() + .describe("A description of the parameter"), + enum: z.array(z.string()).optional().describe("The allowed values"), + }) + .passthrough() + ) + .describe("A map of parameter names to their definitions"), + required: z + .array(z.string()) + .optional() + .describe("The required parameters"), + additionalProperties: z + .boolean() + .optional() + .describe( + "Whether or not additional properties are allowed in the schema" + ), + }) + .passthrough(); + /** * The schema for an OpenAI tool definition * @see https://platform.openai.com/docs/guides/structured-outputs/supported-schemas @@ -9,7 +51,7 @@ import zodToJsonSchema from "zod-to-json-schema"; * allow for extra keys when the zod schema is used for parsing. This is to allow more flexibility for users * to define their own tools according */ -export const openAIToolSchema = z +export const openAIToolDefinitionSchema = z .object({ type: z.literal("function").describe("The type of the tool"), function: z @@ -19,46 +61,8 @@ export const openAIToolSchema = z .string() .optional() .describe("A description of the function"), - parameters: z - .object({ - type: z.literal("object"), - properties: z - .record( - z - .object({ - type: z - .enum([ - "string", - "number", - "boolean", - "object", - "array", - "null", - "integer", - ]) - .describe("The type of the parameter"), - description: z - .string() - .optional() - .describe("A description of the parameter"), - enum: z - .array(z.string()) - .optional() - .describe("The allowed values"), - }) - .passthrough() - ) - .describe("A map of parameter names to their definitions"), - required: z - .array(z.string()) - .optional() - .describe("The required parameters"), - additionalProperties: z - .boolean() - .optional() - .describe( - "Whether or not additional properties are allowed in the schema" - ), + parameters: jsonSchemaZodSchema + .extend({ strict: z .boolean() .optional() @@ -66,7 +70,6 @@ export const openAIToolSchema = z "Whether or not the arguments should exactly match the function definition, only supported for OpenAI models" ), }) - .passthrough() .describe("The parameters that the function accepts"), }) .passthrough() @@ -78,62 +81,139 @@ export const openAIToolSchema = z * The type of a tool definition * @see https://platform.openai.com/docs/guides/structured-outputs/supported-schemas */ -export type OpenAIToolDefinition = z.infer; +export type OpenAIToolDefinition = z.infer; /** * The JSON schema for a tool definition */ -export const openAIToolJSONSchema = zodToJsonSchema(openAIToolSchema, { - removeAdditionalStrategy: "passthrough", -}); +export const openAIToolDefinitionJSONSchema = zodToJsonSchema( + openAIToolDefinitionSchema, + { + removeAdditionalStrategy: "passthrough", + } +); /** - * The schema for an OpenAI tool call, this is what a message that calls a tool looks like - * - * Note: The nested passThrough's are used to allow for extra keys in JSON schema, however, they do not actually - * allow for extra keys when the zod schema is used for parsing. This is to allow more flexibility for users - * to define their own tool calls according + * Anthropic tool format */ -export const openAIToolCallSchema = z.object({ - id: z.string().describe("The ID of the tool call"), - function: z - .object({ - name: z.string().describe("The name of the function"), - arguments: z - .union([z.record(z.unknown()).optional(), z.string()]) - .describe("The arguments for the function"), - }) - .describe("The function that is being called") - .passthrough(), +export const anthropicToolDefinitionSchema = z.object({ + name: z.string(), + description: z.string(), + input_schema: jsonSchemaZodSchema, }); +export type AnthropicToolDefinition = z.infer< + typeof anthropicToolDefinitionSchema +>; + +/** + * -------------------------------- + * Conversion Schemas + * -------------------------------- + */ + +/** + * Parse incoming object as an Anthropic tool call and immediately convert to OpenAI format + */ +export const anthropicToOpenAI = anthropicToolDefinitionSchema.transform( + (anthropic): OpenAIToolDefinition => ({ + id: "", + type: "function", + function: { + name: anthropic.name, + description: anthropic.description, + parameters: anthropic.input_schema, + }, + }) +); + +/** + * Parse incoming object as an OpenAI tool call and immediately convert to Anthropic format + */ +export const openAIToAnthropic = openAIToolDefinitionSchema.transform( + (openai): AnthropicToolDefinition => ({ + name: openai.function.name, + description: openai.function.description ?? openai.function.name, + input_schema: openai.function.parameters, + }) +); + +/** + * -------------------------------- + * Conversion Helpers + * -------------------------------- + */ + /** - * The type of an OpenAI tool call + * Union of all tool call formats * - * @example - * ```typescript - * { - * id: "1", - * function: { - * name: "getCurrentWeather", - * arguments: "{ \"city\": \"San Francisco\" }" - * } - * } - * ``` + * This is useful for functions that need to accept any tool definition format */ -export type OpenAIToolCall = z.infer; +export const anyToolDefinitionSchema = z.union([ + openAIToolDefinitionSchema, + anthropicToolDefinitionSchema, +]); + +export type AnyToolDefinition = z.infer; /** - * The zod schema for multiple OpenAI Tool Calls + * Detect the provider of a tool call object */ -export const openAIToolCallsSchema = z.array(openAIToolCallSchema); +export const detectProvider = ( + toolDefinition: unknown +): { provider: ModelProvider; validatedToolDefinition: AnyToolDefinition } => { + let parsedToolDefinition: z.SafeParseReturnType; + + parsedToolDefinition = openAIToolDefinitionSchema.safeParse(toolDefinition); + if (parsedToolDefinition.success) { + return { + provider: "OPENAI", + validatedToolDefinition: parsedToolDefinition.data, + }; + } + parsedToolDefinition = + anthropicToolDefinitionSchema.safeParse(toolDefinition); + if (parsedToolDefinition.success) { + return { + provider: "ANTHROPIC", + validatedToolDefinition: parsedToolDefinition.data, + }; + } + throw new Error("Unknown tool call format"); +}; /** - * The JSON schema for multiple OpenAI tool calls + * Convert from any tool call format to OpenAI format */ -export const openAIToolCallsJSONSchema = zodToJsonSchema( - openAIToolCallsSchema, - { - removeAdditionalStrategy: "passthrough", +export const toOpenAIFormat = ( + toolDefinition: AnyToolDefinition +): OpenAIToolDefinition => { + const { provider, validatedToolDefinition } = detectProvider(toolDefinition); + switch (provider) { + case "AZURE_OPENAI": + case "OPENAI": + return validatedToolDefinition as OpenAIToolDefinition; + case "ANTHROPIC": + return anthropicToOpenAI.parse(validatedToolDefinition); + default: + assertUnreachable(provider); } -); +}; + +/** + * Convert from OpenAI tool call format to any other format + */ +export const fromOpenAIFormat = ( + toolDefinition: OpenAIToolDefinition, + targetProvider: ModelProvider +): AnyToolDefinition => { + switch (targetProvider) { + case "AZURE_OPENAI": + case "OPENAI": + return toolDefinition; + case "ANTHROPIC": + return openAIToAnthropic.parse(toolDefinition); + default: + assertUnreachable(targetProvider); + } +}; diff --git a/app/src/store/playground/playgroundStore.tsx b/app/src/store/playground/playgroundStore.tsx index 8d3775b7e4..fd270ebcae 100644 --- a/app/src/store/playground/playgroundStore.tsx +++ b/app/src/store/playground/playgroundStore.tsx @@ -7,7 +7,7 @@ import { DEFAULT_CHAT_ROLE, DEFAULT_MODEL_PROVIDER, } from "@phoenix/constants/generativeConstants"; -import { OpenAIToolCall } from "@phoenix/schemas"; +import { OpenAIToolCall } from "@phoenix/schemas/toolCallSchemas"; import { GenAIOperationType, diff --git a/app/src/store/playground/types.ts b/app/src/store/playground/types.ts index e944aa0727..5baa35116d 100644 --- a/app/src/store/playground/types.ts +++ b/app/src/store/playground/types.ts @@ -1,7 +1,7 @@ import { TemplateLanguage } from "@phoenix/components/templateEditor/types"; import { InvocationParameters } from "@phoenix/pages/playground/__generated__/PlaygroundOutputSubscription.graphql"; -import { OpenAIToolCall, OpenAIToolDefinition } from "@phoenix/schemas"; - +import { OpenAIToolDefinition } from "@phoenix/schemas"; +import { OpenAIToolCall } from "@phoenix/schemas/toolCallSchemas"; export type GenAIOperationType = "chat" | "text_completion"; /** * The input mode for the playground