diff --git a/app/jest.config.js b/app/jest.config.js index 50398a7d37..5449658305 100644 --- a/app/jest.config.js +++ b/app/jest.config.js @@ -5,6 +5,8 @@ module.exports = { preset: "ts-jest", testEnvironment: "jsdom", prettierPath: null, + setupFiles: ["/jest.setup.ts"], + testMatch: ["**/__tests__/*.test.ts?(x)"], transform: { "^.+\\.[jt]sx?$": ["esbuild-jest"], }, diff --git a/app/jest.setup.ts b/app/jest.setup.ts new file mode 100644 index 0000000000..73469b8b83 --- /dev/null +++ b/app/jest.setup.ts @@ -0,0 +1,10 @@ +import "jest-canvas-mock"; +jest.mock("@phoenix/config"); + +Object.defineProperty(window, "Config", { + value: { + authenticationEnabled: true, + basename: "/", + platformVersion: "1.0.0", + }, +}); diff --git a/app/package.json b/app/package.json index c055e5e076..90443e8bbc 100644 --- a/app/package.json +++ b/app/package.json @@ -47,6 +47,7 @@ "three-stdlib": "^2.30.4", "use-deep-compare-effect": "^1.8.1", "use-zustand": "^0.0.4", + "zod": "^3.23.8", "zustand": "^4.5.4" }, "devDependencies": { @@ -76,6 +77,7 @@ "eslint-plugin-simple-import-sort": "^10.0.0", "graphql": "^16.9.0", "jest": "^29.7.0", + "jest-canvas-mock": "^2.5.2", "jest-environment-jsdom": "^29.7.0", "only-allow": "^1.2.1", "prettier": "^3.3.3", diff --git a/app/pnpm-lock.yaml b/app/pnpm-lock.yaml index 0b167f7e1e..632ce964cd 100644 --- a/app/pnpm-lock.yaml +++ b/app/pnpm-lock.yaml @@ -134,6 +134,9 @@ importers: use-zustand: specifier: ^0.0.4 version: 0.0.4(react@18.3.1) + zod: + specifier: ^3.23.8 + version: 3.23.8 zustand: specifier: ^4.5.4 version: 4.5.4(@types/react@18.3.10)(react@18.3.1) @@ -216,6 +219,9 @@ importers: jest: specifier: ^29.7.0 version: 29.7.0(@types/node@22.5.4)(babel-plugin-macros@3.1.0) + jest-canvas-mock: + specifier: ^2.5.2 + version: 2.5.2 jest-environment-jsdom: specifier: ^29.7.0 version: 29.7.0 @@ -2275,6 +2281,9 @@ packages: resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} engines: {node: '>= 8'} + cssfontparser@1.2.1: + resolution: {integrity: sha512-6tun4LoZnj7VN6YeegOVb67KBX/7JJsqvj+pv3ZA7F878/eN33AbGa5b/S/wXxS/tcp8nc40xRUrsPlxIyNUPg==} + cssom@0.3.8: resolution: {integrity: sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==} @@ -3263,6 +3272,9 @@ packages: engines: {node: '>=10'} hasBin: true + jest-canvas-mock@2.5.2: + resolution: {integrity: sha512-vgnpPupjOL6+L5oJXzxTxFrlGEIbHdZqFU+LFNdtLxZ3lRDCl17FlTMM7IatoRQkrcyOTMlDinjUguqmQ6bR2A==} + jest-changed-files@29.7.0: resolution: {integrity: sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -3751,6 +3763,9 @@ packages: resolution: {integrity: sha512-8OCq0De/h9ZxseqzCH8Kw/Filf5pF/vMI6+BH7Lu0jXz2pqYCjTAQRolSxRIi+Ax+oCCjlxoJMP0YQ4XlrQNHg==} deprecated: Legacy versions of mkdirp are no longer supported. Please update to mkdirp 1.x. (Note that the API surface has changed to use Promises in 1.x.) + moo-color@1.0.3: + resolution: {integrity: sha512-i/+ZKXMDf6aqYtBhuOcej71YSlbjT3wCO/4H1j8rPvxDJEifdwgg5MaFyu6iYAT8GBZJg2z0dkgK4YMzvURALQ==} + ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} @@ -7516,6 +7531,8 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 + cssfontparser@1.2.1: {} + cssom@0.3.8: {} cssom@0.5.0: {} @@ -8666,6 +8683,11 @@ snapshots: filelist: 1.0.4 minimatch: 3.1.2 + jest-canvas-mock@2.5.2: + dependencies: + cssfontparser: 1.2.1 + moo-color: 1.0.3 + jest-changed-files@29.7.0: dependencies: execa: 5.1.1 @@ -9582,6 +9604,10 @@ snapshots: mkdirp@0.3.5: {} + moo-color@1.0.3: + dependencies: + color-name: 1.1.4 + ms@2.0.0: {} ms@2.1.2: {} diff --git a/app/src/pages/playground/Playground.tsx b/app/src/pages/playground/Playground.tsx index 4fe7b37344..8dd2d6a375 100644 --- a/app/src/pages/playground/Playground.tsx +++ b/app/src/pages/playground/Playground.tsx @@ -8,13 +8,14 @@ import { PlaygroundProvider, usePlaygroundContext, } from "@phoenix/contexts/PlaygroundContext"; +import { InitialPlaygroundState } from "@phoenix/store"; import { PlaygroundInstance } from "./PlaygroundInstance"; import { PlaygroundOperationTypeRadioGroup } from "./PlaygroundOperationTypeRadioGroup"; -export function Playground() { +export function Playground(props: InitialPlaygroundState) { return ( - + { + if (data.span.__typename === "Span") { + return data.span; + } + return null; + }, [data.span]); - return ; + if (!span) { + throw new Error("Span not found"); + } + + const playgroundInstance = useMemo( + () => transformSpanAttributesToPlaygroundInstance(span), + [span] + ); + + return ( + + ); } diff --git a/app/src/pages/playground/__tests__/fixtures.ts b/app/src/pages/playground/__tests__/fixtures.ts new file mode 100644 index 0000000000..6731e2450d --- /dev/null +++ b/app/src/pages/playground/__tests__/fixtures.ts @@ -0,0 +1,40 @@ +import { PlaygroundSpan } from "../spanPlaygroundPageLoader"; + +export const basePlaygroundSpan: PlaygroundSpan = { + __typename: "Span", + context: { + spanId: "test", + }, + attributes: "", +}; +export const spanAttributesWithInputMessages = { + llm: { + output_messages: [ + { + message: { + content: "This is an AI Answer", + role: "assistant", + }, + }, + ], + model_name: "gpt-3.5-turbo", + token_count: { completion: 9.0, prompt: 1881.0, total: 1890.0 }, + input_messages: [ + { + message: { + content: "You are a chatbot", + role: "system", + }, + }, + { + message: { + content: "Anser me the following question. Are you sentient?", + role: "user", + }, + }, + ], + invocation_parameters: + '{"context_window": 16384, "num_output": -1, "is_chat_model": true, "is_function_calling_model": true, "model_name": "gpt-3.5-turbo"}', + }, + openinference: { span: { kind: "LLM" } }, +} as const; diff --git a/app/src/pages/playground/__tests__/playgroundUtils.test.ts b/app/src/pages/playground/__tests__/playgroundUtils.test.ts new file mode 100644 index 0000000000..3dac635514 --- /dev/null +++ b/app/src/pages/playground/__tests__/playgroundUtils.test.ts @@ -0,0 +1,96 @@ +import { _resetInstanceId } from "@phoenix/store"; + +import { + getChatRole, + transformSpanAttributesToPlaygroundInstance, +} from "../playgroundUtils"; + +import { + basePlaygroundSpan, + spanAttributesWithInputMessages, +} from "./fixtures"; + +const expectedPlaygroundInstance = { + id: 0, + activeRunId: null, + isRunning: false, + input: { + variables: {}, + }, + template: { + __type: "chat", + messages: spanAttributesWithInputMessages.llm.input_messages.map( + ({ message }) => message + ), + }, + output: spanAttributesWithInputMessages.llm.output_messages, + tools: undefined, +}; + +describe("transformSpanAttributesToPlaygroundInstance", () => { + beforeEach(() => { + _resetInstanceId(); + }); + it("should throw if the attributes are not parsable", () => { + const span = { + ...basePlaygroundSpan, + attributes: "invalid json", + }; + expect(() => transformSpanAttributesToPlaygroundInstance(span)).toThrow( + "Invalid span attributes, attributes must be valid JSON" + ); + }); + + it("should return null if the attributes do not match the schema", () => { + const span = { + ...basePlaygroundSpan, + attributes: JSON.stringify({}), + }; + expect(transformSpanAttributesToPlaygroundInstance(span)).toBeNull(); + }); + + it("should return a PlaygroundInstance if the attributes contain llm.input_messages", () => { + const span = { + ...basePlaygroundSpan, + attributes: JSON.stringify(spanAttributesWithInputMessages), + }; + + expect(transformSpanAttributesToPlaygroundInstance(span)).toEqual( + expectedPlaygroundInstance + ); + }); + + it("should return a PlaygroundInstance if the attributes contain llm.input_messages, even if output_messages are not present", () => { + const span = { + ...basePlaygroundSpan, + attributes: JSON.stringify({ + ...spanAttributesWithInputMessages, + llm: { + ...spanAttributesWithInputMessages.llm, + output_messages: undefined, + }, + }), + }; + expect(transformSpanAttributesToPlaygroundInstance(span)).toEqual({ + ...expectedPlaygroundInstance, + output: undefined, + }); + }); +}); + +describe("getChatRole", () => { + it("should return the role if it is a valid ChatMessageRole", () => { + expect(getChatRole("user")).toEqual("user"); + }); + + it("should return the ChatMessageRole if the role is included in ChatRoleMap", () => { + expect(getChatRole("assistant")).toEqual("ai"); + // expect(getChatRole("bot")).toEqual("ai"); + // expect(getChatRole("system")).toEqual("system"); + // expect(getChatRole("human:")).toEqual("user"); + }); + + it("should return DEFAULT_CHAT_ROLE if the role is not found", () => { + expect(getChatRole("invalid")).toEqual("user"); + }); +}); diff --git a/app/src/pages/playground/constants.tsx b/app/src/pages/playground/constants.tsx index 64e5a7be52..90ea5993b0 100644 --- a/app/src/pages/playground/constants.tsx +++ b/app/src/pages/playground/constants.tsx @@ -1 +1,16 @@ +import { ChatMessageRole } from "@phoenix/store"; + export const NUM_MAX_PLAYGROUND_INSTANCES = 2; + +export const DEFAULT_CHAT_ROLE = "user"; + +/** + * Map of {@link ChatMessageRole} to potential role values. + * Used to map roles to a canonical role. + */ +export const ChatRoleMap: Record = { + user: ["user", "human"], + ai: ["assistant", "bot", "ai"], + system: ["system"], + tool: ["tool"], +}; diff --git a/app/src/pages/playground/playgroundUtils.ts b/app/src/pages/playground/playgroundUtils.ts index 50a154f384..b28f9775d7 100644 --- a/app/src/pages/playground/playgroundUtils.ts +++ b/app/src/pages/playground/playgroundUtils.ts @@ -1,7 +1,13 @@ +import { generateInstanceId, PlaygroundInstance } from "@phoenix/store"; import { ChatMessageRole, chatMessageRoles, } from "@phoenix/store/playgroundStore"; +import { safelyParseJSON } from "@phoenix/utils/jsonUtils"; + +import { ChatRoleMap, DEFAULT_CHAT_ROLE } from "./constants"; +import { llmAttributesSchema } from "./schemas"; +import { PlaygroundSpan } from "./spanPlaygroundPageLoader"; /** * Checks if a string is a valid chat message role @@ -9,3 +15,60 @@ import { export function isChatMessageRole(role: unknown): role is ChatMessageRole { return chatMessageRoles.includes(role as ChatMessageRole); } + +/** + * Takes a string role and attempts to map the role to a valid ChatMessageRole. + * If the role is not found, it will default to {@link DEFAULT_CHAT_ROLE}. + * @param role the role to map + * @returns ChatMessageRole + * + * NB: Only exported for testing + */ +export function getChatRole(role: string): ChatMessageRole { + if (isChatMessageRole(role)) { + return role; + } + + for (const [chatRole, acceptedValues] of Object.entries(ChatRoleMap)) { + if (acceptedValues.includes(role)) { + return chatRole as ChatMessageRole; + } + } + return DEFAULT_CHAT_ROLE; +} + +export function transformSpanAttributesToPlaygroundInstance( + span: PlaygroundSpan +): PlaygroundInstance | null { + const { json: parsedAttributes, parseError } = safelyParseJSON( + span.attributes + ); + if (parseError) { + throw new Error("Invalid span attributes, attributes must be valid JSON"); + } + const { data, success } = llmAttributesSchema.safeParse(parsedAttributes); + if (!success) { + return null; + } + // TODO(parker): add support for tools, variables, and input / output variants + // https://github.com/Arize-ai/phoenix/issues/4886 + return { + id: generateInstanceId(), + activeRunId: null, + isRunning: false, + input: { + variables: {}, + }, + template: { + __type: "chat", + messages: data.llm.input_messages.map(({ message }) => { + return { + role: getChatRole(message.role), + content: message.content, + }; + }), + }, + output: data.llm.output_messages, + tools: undefined, + }; +} diff --git a/app/src/pages/playground/schemas.ts b/app/src/pages/playground/schemas.ts new file mode 100644 index 0000000000..f9f964f4fa --- /dev/null +++ b/app/src/pages/playground/schemas.ts @@ -0,0 +1,72 @@ +import { z } from "zod"; + +import { + ImageAttributesPostfixes, + LLMAttributePostfixes, + MessageAttributePostfixes, + MessageContentsAttributePostfixes, + SemanticAttributePrefixes, +} from "@arizeai/openinference-semantic-conventions"; + +/** + * The zod schema for llm tool calls in an input message + * @see {@link https://github.com/Arize-ai/openinference/blob/main/spec/semantic_conventions.md|Semantic Conventions} + */ +const toolCallSchema = z + .object({ + function: z + .object({ + name: z.string(), + arguments: z.string(), + }) + .partial(), + }) + .partial(); + +/** + * The zod schema for llm message contents + * @see {@link https://github.com/Arize-ai/openinference/blob/main/spec/semantic_conventions.md|Semantic Conventions} + */ +const messageContentSchema = z.object({ + [SemanticAttributePrefixes.message_content]: z + .object({ + [MessageContentsAttributePostfixes.type]: z.string(), + [MessageContentsAttributePostfixes.text]: z.string(), + [MessageContentsAttributePostfixes.image]: z + .object({ + [MessageContentsAttributePostfixes.image]: z + .object({ + [ImageAttributesPostfixes.url]: z.string(), + }) + .partial(), + }) + .partial(), + }) + .partial(), +}); + +/** + * The zod schema for llm messages + * @see {@link https://github.com/Arize-ai/openinference/blob/main/spec/semantic_conventions.md|Semantic Conventions} + */ +const messageSchema = z.object({ + [SemanticAttributePrefixes.message]: z.object({ + [MessageAttributePostfixes.role]: z.string(), + [MessageAttributePostfixes.content]: z.string(), + [MessageAttributePostfixes.name]: z.string().optional(), + [MessageAttributePostfixes.tool_calls]: z.array(toolCallSchema).optional(), + [MessageAttributePostfixes.contents]: z + .array(messageContentSchema) + .optional(), + }), +}); +/** + * The zod schema for llm attributes + * @see {@link https://github.com/Arize-ai/openinference/blob/main/spec/semantic_conventions.md|Semantic Conventions} + */ +export const llmAttributesSchema = z.object({ + [SemanticAttributePrefixes.llm]: z.object({ + [LLMAttributePostfixes.input_messages]: z.array(messageSchema), + [LLMAttributePostfixes.output_messages]: z.optional(z.array(messageSchema)), + }), +}); diff --git a/app/src/pages/playground/spanPlaygroundPageLoader.ts b/app/src/pages/playground/spanPlaygroundPageLoader.ts index 792e0ef46b..84efdd04b3 100644 --- a/app/src/pages/playground/spanPlaygroundPageLoader.ts +++ b/app/src/pages/playground/spanPlaygroundPageLoader.ts @@ -3,7 +3,20 @@ import { LoaderFunctionArgs } from "react-router"; import RelayEnvironment from "@phoenix/RelayEnvironment"; -import { spanPlaygroundPageLoaderQuery } from "./__generated__/spanPlaygroundPageLoaderQuery.graphql"; +import { + spanPlaygroundPageLoaderQuery, + spanPlaygroundPageLoaderQuery$data, +} from "./__generated__/spanPlaygroundPageLoaderQuery.graphql"; + +/** + * The type of a span that is fetched to pre-populate the playground. + * This span gets fetched when navigating from a span to the playground, used for span replay. + */ +export type PlaygroundSpan = Extract< + spanPlaygroundPageLoaderQuery$data["span"], + { __typename: "Span" } +>; + export async function spanPlaygroundPageLoader(args: LoaderFunctionArgs) { const { spanId } = args.params; if (!spanId || typeof spanId !== "string") { diff --git a/app/src/pages/trace/SpanDetails.tsx b/app/src/pages/trace/SpanDetails.tsx index 755b4fc749..ab9af7fffc 100644 --- a/app/src/pages/trace/SpanDetails.tsx +++ b/app/src/pages/trace/SpanDetails.tsx @@ -80,6 +80,7 @@ import { isAttributeMessages, } from "@phoenix/openInference/tracing/types"; import { assertUnreachable, isStringArray } from "@phoenix/typeUtils"; +import { safelyParseJSON } from "@phoenix/utils/jsonUtils"; import { formatFloat, numberFormatter } from "@phoenix/utils/numberFormatUtils"; import { RetrievalEvaluationLabel } from "../project/RetrievalEvaluationLabel"; @@ -118,11 +119,7 @@ const useSafelyParsedJSON = ( jsonStr: string ): { json: { [key: string]: unknown } | null; parseError?: unknown } => { return useMemo(() => { - try { - return { json: JSON.parse(jsonStr) }; - } catch (e) { - return { json: null, parseError: e }; - } + return safelyParseJSON(jsonStr); }, [jsonStr]); }; diff --git a/app/src/store/playgroundStore.tsx b/app/src/store/playgroundStore.tsx index 8315720155..af39ff8a12 100644 --- a/app/src/store/playgroundStore.tsx +++ b/app/src/store/playgroundStore.tsx @@ -6,6 +6,20 @@ export type GenAIOperationType = "chat" | "text_completion"; let playgroundInstanceIdIndex = 0; let playgroundRunIdIndex = 0; +/** + * Generates a new playground instance ID + */ +export const generateInstanceId = () => playgroundInstanceIdIndex++; + +/** + * Resets the playground instance ID to 0 + * + * NB: This is only used for testing purposes + */ +export const _resetInstanceId = () => { + playgroundInstanceIdIndex = 0; +}; + /** * The input mode for the playground * @example "manual" or "dataset" @@ -73,6 +87,8 @@ export interface PlaygroundProps { instances: Array; } +export type InitialPlaygroundState = Partial; + type DatasetInput = { datasetId: string; }; @@ -178,7 +194,7 @@ const DEFAULT_TEXT_COMPLETION_TEMPLATE: PlaygroundTextCompletionTemplate = { }; export const createPlaygroundStore = ( - initialProps?: Partial + initialProps?: InitialPlaygroundState ) => { const playgroundStore: StateCreator = (set, get) => ({ operationType: "chat", @@ -186,7 +202,7 @@ export const createPlaygroundStore = ( setInputMode: (inputMode: PlaygroundInputMode) => set({ inputMode }), instances: [ { - id: playgroundInstanceIdIndex++, + id: generateInstanceId(), template: DEFAULT_CHAT_COMPLETION_TEMPLATE, tools: {}, input: { variables: {} }, @@ -201,7 +217,7 @@ export const createPlaygroundStore = ( set({ instances: [ { - id: playgroundInstanceIdIndex++, + id: generateInstanceId(), template: DEFAULT_CHAT_COMPLETION_TEMPLATE, tools: {}, input: { variables: {} }, @@ -215,7 +231,7 @@ export const createPlaygroundStore = ( set({ instances: [ { - id: playgroundInstanceIdIndex++, + id: generateInstanceId(), template: DEFAULT_TEXT_COMPLETION_TEMPLATE, tools: {}, input: { variables: {} }, @@ -239,7 +255,7 @@ export const createPlaygroundStore = ( instance, { ...instance, - id: playgroundInstanceIdIndex++, + id: generateInstanceId(), activeRunId: null, }, ], diff --git a/app/src/utils/jsonUtils.ts b/app/src/utils/jsonUtils.ts index 395388f3e2..792f76bc1d 100644 --- a/app/src/utils/jsonUtils.ts +++ b/app/src/utils/jsonUtils.ts @@ -32,3 +32,11 @@ export function isJSONString({ export function isJSONObjectString(str: string) { return isJSONString({ str, excludeArray: true, excludePrimitives: true }); } + +export function safelyParseJSON(str: string) { + try { + return { json: JSON.parse(str) }; + } catch (e) { + return { json: null, parseError: e }; + } +}