diff --git a/package-lock.json b/package-lock.json index 4066336e89..2f8198536f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2290,7 +2290,6 @@ "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", @@ -4018,7 +4017,6 @@ "integrity": "sha512-b7W4snvXYi1T2puUjxamASCCNhNzVSzb/fQUuGSkdjm/AFfJ24jo8kOHQyOcaoArCG71sVQci4vkZaITzl/V1w==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@cucumber/ci-environment": "10.0.1", "@cucumber/cucumber-expressions": "18.0.1", @@ -4181,7 +4179,6 @@ "integrity": "sha512-659CCFsrsyvuBi/Eix1fnhSheMnojSfnBcqJ3IMPNawx7JlrNJDcXYSSdxcUw3n/nG05P+ptCjmiZY3i14p+tA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@cucumber/messages": ">=19.1.4 <29" } @@ -4360,7 +4357,6 @@ "integrity": "sha512-Kxap9uP5jD8tHUZVjTWgzxemi/0uOsbGjd4LBOSxcJoOCRbESFwemUzilJuzNTB8pcTQUh8D5oudUyxfkJOKmA==", "dev": true, "license": "MIT", - "peer": true, "peerDependencies": { "@cucumber/messages": ">=17.1.1" } @@ -4371,7 +4367,6 @@ "integrity": "sha512-2LzZtOwYKNlCuNf31ajkrekoy2M4z0Z1QGiPH40n4gf5t8VOUFb7m1ojtR4LmGvZxBGvJZP8voOmRqDWzBzYKA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/uuid": "10.0.0", "class-transformer": "0.5.1", @@ -7265,7 +7260,6 @@ "dev": true, "hasInstallScript": true, "license": "MIT", - "peer": true, "dependencies": { "@napi-rs/wasm-runtime": "0.2.4", "@yarnpkg/lockfile": "^1.1.0", @@ -7601,7 +7595,6 @@ "integrity": "sha512-pzGXp14KF2Q4CDZGQgPK4l8zEg7i6cNkb+10yc8ZA5K41cLe3ZbWW1YxtY2e/glHauOJwTLSVjH4tiRVtOTizg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "iterare": "1.2.1", "tslib": "2.8.1", @@ -7684,7 +7677,6 @@ "integrity": "sha512-UVSf0yaWFBC2Zn2FOWABXxCnyG8XNIXrNnvTFpbUFqaJu1YDdwJ7wQBBqxq9CtJT7ILqSmfhOU7HS0d/0EAxpw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "cors": "2.8.5", "express": "5.0.1", @@ -9222,7 +9214,6 @@ "integrity": "sha512-/g2d4sW9nUDJOMz3mabVQvOGhVa4e/BN/Um7yca9Bb2XTzPPnfTWHWQg+IsEYO7M3Vx+EXvaM/I2pJWIMun1bg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@octokit/auth-token": "^4.0.0", "@octokit/graphql": "^7.1.0", @@ -9411,7 +9402,6 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=8.0.0" } @@ -10498,7 +10488,6 @@ "integrity": "sha512-EI0Ti5pojD2p7TmcS7RRa+AJVahdQvP/urpcSbK/K9Rlk6+dwMJTQ354pCNGCwfke8x4yKr5+iH85wcERSkwLQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "cluster-key-slot": "1.1.2" }, @@ -12502,7 +12491,6 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-22.17.1.tgz", "integrity": "sha512-y3tBaz+rjspDTylNjAX37jEC3TETEFGNJL6uQDxwF9/8GLLIjW1rvVHlynyuUKMnMr1Roq8jOv3vkopBjC4/VA==", "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~6.21.0" } @@ -12584,7 +12572,6 @@ "integrity": "sha512-I98SaDCar5lvEYl80ClRIUztH/hyWHR+I2f+5yTVp/MQ205HgYkA2b5mVdry/+nsEIrf8I65KA5V/PASx68MsQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/prop-types": "*", "@types/scheduler": "^0.16", @@ -12860,7 +12847,6 @@ "integrity": "sha512-gTtSdWX9xiMPA/7MV9STjJOOYtWwIJIYxkQxnSV1U3xcE+mnJSH3f6zI0RYP+ew66WSlZ5ed+h0VCxsvdC1jJg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.41.0", "@typescript-eslint/types": "8.41.0", @@ -14069,7 +14055,6 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -14199,7 +14184,6 @@ "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -14938,7 +14922,6 @@ "integrity": "sha512-Cg7TFGpIr01vOQNODXOOaGz2NpCU5gl8x1qJFbb6hbZxR7XrcE2vtbAsTAbJ7/xwJtUuJEw8K8Zr/AE0LHlesg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/runtime": "^7.12.5", "cosmiconfig": "^7.0.0", @@ -15445,7 +15428,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.19", "caniuse-lite": "^1.0.30001751", @@ -17551,8 +17533,7 @@ "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1312386.tgz", "integrity": "sha512-DPnhUXvmvKT2dFA/j7B+riVLUt9Q6RKJlcppojL5CoRywJJKLDYnRlw0gTFKfgDPHP5E04UoB71SxoJlVZy8FA==", "dev": true, - "license": "BSD-3-Clause", - "peer": true + "license": "BSD-3-Clause" }, "node_modules/di": { "version": "0.0.1", @@ -18318,7 +18299,6 @@ "integrity": "sha512-RNCHRX5EwdrESy3Jc9o8ie8Bog+PeYvvSR8sDGoZxNFTvZ4dlxUB3WzQ3bQMztFrSRODGrLLj8g6OFuGY/aiQg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.12.1", @@ -24209,7 +24189,6 @@ "dev": true, "hasInstallScript": true, "license": "MIT", - "peer": true, "dependencies": { "@napi-rs/wasm-runtime": "0.2.4", "@yarnpkg/lockfile": "^1.1.0", @@ -28147,7 +28126,6 @@ "dev": true, "hasInstallScript": true, "license": "MIT", - "peer": true, "dependencies": { "@napi-rs/wasm-runtime": "0.2.4", "@yarnpkg/lockfile": "^1.1.0", @@ -29674,7 +29652,6 @@ "integrity": "sha512-7bdYcv7V6U3KAtWjpQJJBww0UEsWuh4yQ/EjNf2HeO/NnvKjpvhEIe/A/TleP6wtmSKnUnghs5A9jUoK6iDdkA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "buffer-writer": "2.0.0", "packet-reader": "1.0.0", @@ -30571,7 +30548,6 @@ "integrity": "sha512-gnhPt75i/dq/z3/6q/0asP78D0u592D5L1pd7M8P+dck6Fu/jJeL6iVVK23fptSUZj8Vjf++7wXA8UNclGQcbA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0", "object-assign": "^4.1.1" @@ -31070,8 +31046,7 @@ "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==", "dev": true, - "license": "Apache-2.0", - "peer": true + "license": "Apache-2.0" }, "node_modules/reflect.getprototypeof": { "version": "1.0.10", @@ -31706,7 +31681,6 @@ "integrity": "sha512-3GuObel8h7Kqdjt0gxkEzaifHTqLVW56Y/bjN7PSQtkKr0w3V/QYSdt6QWYtd7A1xUtYQigtdUfgj1RvWVtorw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/estree": "1.0.8" }, @@ -31826,7 +31800,6 @@ "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "tslib": "^2.1.0" } @@ -34988,7 +34961,6 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -35565,7 +35537,6 @@ "integrity": "sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -36134,7 +36105,6 @@ "integrity": "sha512-7h/weGm9d/ywQ6qzJ+Xy+r9n/3qgp/thalBbpOi5i223dPXKi04IBtqPN9nTd+jBc7QKfvDbaBnFipYp4sJAUQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", @@ -36184,7 +36154,6 @@ "integrity": "sha512-MfwFQ6SfwinsUVi0rNJm7rHZ31GyTcpVE5pgVA3hwFRb7COD4TzjUUwhGWKfO50+xdc2MQPuEBBJoqIMGt3JDw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@discoveryjs/json-ext": "^0.6.1", "@webpack-cli/configtest": "^3.0.1", @@ -37125,8 +37094,7 @@ "version": "0.15.1", "resolved": "https://registry.npmjs.org/zone.js/-/zone.js-0.15.1.tgz", "integrity": "sha512-XE96n56IQpJM7NAoXswY3XRLcWFW83xe0BiAOeMD7K5k5xecOeul3Qcpx6GqEeeHNkW5DWL5zOyTbEfB4eti8w==", - "license": "MIT", - "peer": true + "license": "MIT" }, "packages/auto-configuration-propagators": { "name": "@opentelemetry/auto-configuration-propagators", diff --git a/packages/instrumentation-openai/README.md b/packages/instrumentation-openai/README.md index 0b5fd9259e..9a88c40508 100644 --- a/packages/instrumentation-openai/README.md +++ b/packages/instrumentation-openai/README.md @@ -39,7 +39,7 @@ registerInstrumentations({ ## Semantic Conventions -This package implements Semantic Convention [Version 1.36.0](https://github.com/open-telemetry/semantic-conventions/blob/v1.36.0/docs/README.md). +This package implements Semantic Convention [Version 1.36.0](https://github.com/open-telemetry/semantic-conventions/blob/v1.36.0/docs/README.md) for Chat Completions, and Semantic Convention [Version 1.38.0](https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/README.md) for Responses API. ## Useful links diff --git a/packages/instrumentation-openai/src/instrumentation.ts b/packages/instrumentation-openai/src/instrumentation.ts index 492c1c1d16..ff9f211ddc 100644 --- a/packages/instrumentation-openai/src/instrumentation.ts +++ b/packages/instrumentation-openai/src/instrumentation.ts @@ -23,7 +23,7 @@ import { InstrumentationBase, InstrumentationNodeModuleDefinition, } from '@opentelemetry/instrumentation'; -import { SeverityNumber } from '@opentelemetry/api-logs'; +import { type AnyValue, SeverityNumber } from '@opentelemetry/api-logs'; import type { ChatCompletion, ChatCompletionMessageToolCall, @@ -40,10 +40,28 @@ import type { Embeddings, EmbeddingCreateParams, } from 'openai/resources/embeddings'; +import type { + Responses, + Response, + EasyInputMessage, + ResponseCodeInterpreterToolCall, + ResponseComputerToolCall, + ResponseCreateParams, + ResponseCustomToolCall, + ResponseCustomToolCallOutput, + ResponseFileSearchToolCall, + ResponseFunctionToolCall, + ResponseFunctionWebSearch, + ResponseOutputItem, + ResponseOutputMessage, + ResponseReasoningItem, + ResponseStreamEvent, + ResponseInputItem, +} from 'openai/resources/responses/responses'; import type { Stream } from 'openai/streaming'; - import { ATTR_EVENT_NAME, + ATTR_GEN_AI_CONVERSATION_ID, ATTR_GEN_AI_OPERATION_NAME, ATTR_GEN_AI_REQUEST_ENCODING_FORMATS, ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY, @@ -57,17 +75,32 @@ import { ATTR_GEN_AI_RESPONSE_ID, ATTR_GEN_AI_RESPONSE_MODEL, ATTR_GEN_AI_SYSTEM, + ATTR_GEN_AI_PROVIDER_NAME, ATTR_GEN_AI_TOKEN_TYPE, ATTR_GEN_AI_USAGE_INPUT_TOKENS, ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, + ATTR_GEN_AI_INPUT_MESSAGES, + ATTR_GEN_AI_OUTPUT_MESSAGES, METRIC_GEN_AI_CLIENT_OPERATION_DURATION, METRIC_GEN_AI_CLIENT_TOKEN_USAGE, + EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS, + EVENT_GEN_AI_CHOICE, + EVENT_GEN_AI_SYSTEM_MESSAGE, + EVENT_GEN_AI_USER_MESSAGE, + EVENT_GEN_AI_ASSISTANT_MESSAGE, + EVENT_GEN_AI_TOOL_MESSAGE, + GEN_AI_TOKEN_TYPE_VALUE_INPUT, + GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + GEN_AI_OPERATION_NAME_VALUE_CHAT, + GEN_AI_OPERATION_NAME_VALUE_EMBEDDINGS, + GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + ATTR_GEN_AI_SYSTEM_INSTRUCTIONS } from './semconv'; /** @knipignore */ import { PACKAGE_NAME, PACKAGE_VERSION } from './version'; import { getEnvBool, getAttrsFromBaseURL } from './utils'; -import { OpenAIInstrumentationConfig } from './types'; -import { +import type { OpenAIInstrumentationConfig } from './types'; +import type { APIPromise, GenAIMessage, GenAIChoiceEventBody, @@ -76,16 +109,16 @@ import { GenAIAssistantMessageEventBody, GenAIToolMessageEventBody, GenAIToolCall, + GenericPart, + MessagePart, + OutputMessages, + TextPart, + ToolCallRequestPart, + ToolCallResponsePart, + ChatMessage, + InputMessages, } from './internal-types'; -// The JS semconv package doesn't yet emit constants for event names. -// TODO: otel-js issue for semconv pkg not including event names -const EVENT_GEN_AI_SYSTEM_MESSAGE = 'gen_ai.system.message'; -const EVENT_GEN_AI_USER_MESSAGE = 'gen_ai.user.message'; -const EVENT_GEN_AI_ASSISTANT_MESSAGE = 'gen_ai.assistant.message'; -const EVENT_GEN_AI_TOOL_MESSAGE = 'gen_ai.tool.message'; -const EVENT_GEN_AI_CHOICE = 'gen_ai.choice'; - export class OpenAIInstrumentation extends InstrumentationBase { private _genaiClientOperationDuration!: Histogram; private _genaiClientTokenUsage!: Histogram; @@ -117,7 +150,11 @@ export class OpenAIInstrumentation extends InstrumentationBase=4.19.0 <7'], - modExports => { + module => { + const modExports = + module[Symbol.toStringTag] === 'Module' + ? module.default // ESM + : module; // CommonJS this._wrap( modExports.OpenAI.Chat.Completions.prototype, 'create', @@ -128,12 +165,22 @@ export class OpenAIInstrumentation extends InstrumentationBase { + module => { + const modExports = + module[Symbol.toStringTag] === 'Module' + ? module.default // ESM + : module; // CommonJS this._unwrap(modExports.OpenAI.Chat.Completions.prototype, 'create'); this._unwrap(modExports.OpenAI.Embeddings.prototype, 'create'); + this._unwrap(modExports.OpenAI.Responses.prototype, 'create'); } ), ]; @@ -169,6 +216,7 @@ export class OpenAIInstrumentation extends InstrumentationBase { @@ -177,7 +225,7 @@ export class OpenAIInstrumentation extends InstrumentationBase ) { - if (!self.isEnabled) { + if (!self.isEnabled()) { return original.apply(this, args); } @@ -186,7 +234,7 @@ export class OpenAIInstrumentation extends InstrumentationBase; try { startInfo = self._startChatCompletionsSpan( params, @@ -258,9 +306,9 @@ export class OpenAIInstrumentation extends InstrumentationBase, + iterator: AsyncIterator, span: Span, startNow: number, config: OpenAIInstrumentationConfig, commonAttrs: Attributes, ctx: Context ) { - let id; - let model; + const iterable = { [Symbol.asyncIterator]: () => iterator }; + let id: string | undefined; + let model: string | undefined; const finishReasons: string[] = []; const choices = []; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - for await (const chunk of streamIter as any) { + for await (const chunk of iterable) { yield chunk; // Gather telemetry from this chunk. @@ -542,12 +588,12 @@ export class OpenAIInstrumentation extends InstrumentationBase { @@ -742,7 +789,7 @@ export class OpenAIInstrumentation extends InstrumentationBase ) { - if (!self.isEnabled) { + if (!self.isEnabled()) { return original.apply(this, args); } @@ -750,7 +797,7 @@ export class OpenAIInstrumentation extends InstrumentationBase; try { startInfo = self._startEmbeddingsSpan(params, this?._client?.baseURL); } catch (err) { @@ -784,9 +831,9 @@ export class OpenAIInstrumentation extends InstrumentationBase { + // https://platform.openai.com/docs/api-reference/responses/create + return function patchedCreate( + this: Responses, + ...args: Parameters + ) { + if (!self.isEnabled()) { + return original.apply(this, args); + } + + self._diag.debug('OpenAI.Responses.create args: %O', args); + const params = args[0]; + const config = self.getConfig(); + const startNow = performance.now(); + + let startInfo: ReturnType; + try { + startInfo = self._startResponsesSpan( + params, + config, + this?._client?.baseURL + ); + } catch (err) { + self._diag.error('unexpected error starting span:', err); + return original.apply(this, args); + } + const { span, ctx, commonAttrs } = startInfo; + + const apiPromise = context.with(ctx, () => original.apply(this, args)); + + // Streaming. + if (isStreamPromise(params, apiPromise)) { + return apiPromise.then(stream => { + self._wrap(stream as Stream, Symbol.asyncIterator, origIterator => { + return () => { + return self._onResponsesStreamIterator( + origIterator.call(stream), + span, + startNow, + config, + commonAttrs, + ctx + ); + }; + }); + return stream; + }); + } + + // Non-streaming. + apiPromise + .then(result => { + self._onResponsesCreateResult( + span, + startNow, + commonAttrs, + result as Response, + config, + ctx, + ); + }) + .catch( + self._createAPIPromiseRejectionHandler(startNow, span, commonAttrs) + ); + + return apiPromise; + }; + }; + } + + _startResponsesSpan( + params: ResponseCreateParams, + config: OpenAIInstrumentationConfig, + baseURL: string | undefined + ) { + // Common attributes for the span, metrics, and log events. + const commonAttrs: Attributes = Object.assign({ + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: params.model, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_SYSTEM_INSTRUCTIONS]: params.instructions, + }, getAttrsFromBaseURL(baseURL, this._diag)); + + // Span attributes. + const attrs: Attributes = Object.assign({ + [ATTR_GEN_AI_REQUEST_MAX_TOKENS]: params.max_output_tokens ?? undefined, + [ATTR_GEN_AI_REQUEST_TEMPERATURE]: params.temperature ?? undefined, + [ATTR_GEN_AI_REQUEST_TOP_P]: params.top_p ?? undefined, + }, commonAttrs); + + const span: Span = this.tracer.startSpan( + `${attrs[ATTR_GEN_AI_OPERATION_NAME]} ${attrs[ATTR_GEN_AI_REQUEST_MODEL]}`, + { + kind: SpanKind.CLIENT, + attributes: attrs, + } + ); + const ctx: Context = trace.setSpan(context.active(), span); + + const inputs: InputMessages = new ConvertResponseInputsToInputMessagesUseCase( + config.captureMessageContent + ).convert(params); + + // Capture inputs as log events. + this.logger.emit({ + timestamp: Date.now(), + context: ctx, + severityNumber: SeverityNumber.INFO, + eventName: EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS, + attributes: { + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined // inputs as AnyValue, + }, + body: inputs as AnyValue, + }); + + return { span, ctx, commonAttrs }; + } + + async *_onResponsesStreamIterator( + iterator: AsyncIterator, + span: Span, + startNow: number, + config: OpenAIInstrumentationConfig, + commonAttrs: Attributes, + ctx: Context + ) { + const iterable = { [Symbol.asyncIterator]: () => iterator }; + let model: string | undefined; + const converter = new ConvertResponseOutputsToOutputMessagesUseCase(config.captureMessageContent); + + for await (const event of iterable) { + yield event; + + // Gather telemetry from this chunk. + this._diag.debug( + 'OpenAI.Responses.create stream event: %O', + event + ); + + switch (event.type) { + case 'response.created': { + const response = event.response; + model = response.model; + span.setAttributes({ + [ATTR_GEN_AI_RESPONSE_ID]: response.id, + [ATTR_GEN_AI_RESPONSE_MODEL]: model, + [ATTR_GEN_AI_CONVERSATION_ID]: response.conversation?.id + }); + break; + } + case 'response.output_item.done': { + const output = converter.convert([event.item]); + span.setAttribute(ATTR_GEN_AI_RESPONSE_FINISH_REASONS, [output[0].finish_reason]) + this.logger.emit({ + timestamp: Date.now(), + context: ctx, + severityNumber: SeverityNumber.INFO, + eventName: EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS, + attributes: { + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined // output as AnyValue, + }, + body: output as AnyValue, + }); + break; + } + case 'response.completed': { + const usage = event.response.usage; + span.setAttributes({ + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: usage?.input_tokens, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: usage?.output_tokens, + }); + if (usage?.input_tokens) { + this._genaiClientTokenUsage.record(usage?.input_tokens, { + ...commonAttrs, + [ATTR_GEN_AI_RESPONSE_MODEL]: model, + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }); + } + if (usage?.output_tokens) { + this._genaiClientTokenUsage.record(usage?.output_tokens, { + ...commonAttrs, + [ATTR_GEN_AI_RESPONSE_MODEL]: model, + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }); + } + break; + } + } + } + + this._genaiClientOperationDuration.record( + (performance.now() - startNow) / 1000, + { + ...commonAttrs, + [ATTR_GEN_AI_RESPONSE_MODEL]: model, + } + ); + + span.end(); + } + + _onResponsesCreateResult( + span: Span, + startNow: number, + commonAttrs: Attributes, + result: Response, + config: OpenAIInstrumentationConfig, + ctx: Context, + ) { + this._diag.debug('OpenAI.Responses.create result: %O', result); + const { id, model, conversation, output, usage } = result; + try { + if (conversation) { + span.setAttribute( + ATTR_GEN_AI_CONVERSATION_ID, + conversation.id + ); + } + span.setAttribute(ATTR_GEN_AI_RESPONSE_ID, id); + span.setAttribute(ATTR_GEN_AI_RESPONSE_MODEL, model); + if (usage) { + span.setAttribute(ATTR_GEN_AI_USAGE_INPUT_TOKENS, usage.input_tokens); + span.setAttribute(ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, usage.output_tokens); + this._genaiClientTokenUsage.record(usage.input_tokens, { + ...commonAttrs, + [ATTR_GEN_AI_RESPONSE_MODEL]: model, + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }); + + this._genaiClientTokenUsage.record(usage.output_tokens, { + ...commonAttrs, + [ATTR_GEN_AI_RESPONSE_MODEL]: model, + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }); + } + + const outputs = new ConvertResponseOutputsToOutputMessagesUseCase( + config.captureMessageContent, + ).convert(output); + + span.setAttribute(ATTR_GEN_AI_RESPONSE_FINISH_REASONS, [outputs[0].finish_reason]) + + // Capture outputs as a log event. + this.logger.emit({ + timestamp: Date.now(), + context: ctx, + severityNumber: SeverityNumber.INFO, + eventName: EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS, + attributes: { + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: outputs as AnyValue, + }, + body: outputs as AnyValue, + }); + + this._genaiClientOperationDuration.record( + (performance.now() - startNow) / 1000, + { + ...commonAttrs, + [ATTR_GEN_AI_RESPONSE_MODEL]: model, + } + ); + } catch (err) { + this._diag.error( + 'unexpected error getting telemetry from chat result:', + err + ); + } + span.end(); + } +} + +class ConvertResponseInputsToInputMessagesUseCase { + constructor(private readonly captureMessageContent = false) { } + + convert(params: ResponseCreateParams): InputMessages { + const messages: Array = []; + + if (typeof params.instructions === 'string') { + messages.push(this.message({ role: 'system', content: params.instructions })); + } + if (typeof params.input === 'string') { + messages.push(this.message({ role: 'user', content: params.input })); + } else if (Array.isArray(params.input)) { + messages.push(...params.input.map((input): ChatMessage => this[input.type ?? 'message'](input as never))); + } + + return messages; + } + + message( + item: + | EasyInputMessage + | ResponseInputItem.Message + | Responses.ResponseInputMessageItem + | ResponseOutputMessage, + ): ChatMessage { + const parts: Array = []; + if (typeof item.content === 'string') { + if (this.captureMessageContent) { + parts.push({ + type: 'text', + content: item.content, + } satisfies TextPart); + } else { + parts.push({ + type: 'text', + content: undefined, + } satisfies GenericPart); + } + } else if (Array.isArray(item.content)) { + for (const content of item.content) { + switch (content.type) { + case 'input_text': + case 'output_text': + if (this.captureMessageContent) { + parts.push({ + type: 'text', + content: content.text, + } satisfies TextPart); + } else { + parts.push({ + type: 'text', + content: undefined, + } satisfies GenericPart); + } + break; + case 'refusal': + parts.push({ + type: 'refusal', + refusal: content.refusal, + } satisfies GenericPart); + break; + case 'input_image': + parts.push({ + ...(this.captureMessageContent ? content : undefined), + type: 'image', + } satisfies GenericPart); + break; + case 'input_file': + parts.push({ + ...(this.captureMessageContent ? content : undefined), + type: 'file', + } satisfies GenericPart); + break; + case 'input_audio': { + parts.push({ + ...(this.captureMessageContent ? content : undefined), + type: 'audio', + } satisfies GenericPart); + break; + } + } + } + } + + return { + role: item.role, + parts, + } satisfies ChatMessage; + } + + function_call(item: ResponseFunctionToolCall): ChatMessage { + return { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: item.id, + name: item.name, + arguments: this.captureMessageContent ? item.arguments : undefined, + call_id: item.call_id, + } satisfies ToolCallRequestPart, + ], + } satisfies ChatMessage; + } + + custom_tool_call(item: ResponseCustomToolCall): ChatMessage { + return { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: item.id, + name: item.name, + arguments: this.captureMessageContent ? item.input : undefined, + call_id: item.call_id, + } satisfies ToolCallRequestPart, + ], + } satisfies ChatMessage; + } + + reasoning(item: ResponseReasoningItem): ChatMessage { + const parts: Array = []; + for (const summary of item.summary) { + parts.push({ + type: item.type, + text: this.captureMessageContent ? summary.text : undefined, + }); + } + if (item.content) { + for (const content of item.content) { + parts.push({ + type: item.type, + text: this.captureMessageContent ? content.text : undefined, + }); + } + } + + return { + role: 'assistant', + parts, + } satisfies ChatMessage; + } + + file_search_call(item: ResponseFileSearchToolCall): ChatMessage { + const parts: Array = [ + { + type: 'tool_call', + id: item.id, + name: item.type, + arguments: this.captureMessageContent ? item.queries : undefined, + } satisfies ToolCallRequestPart, + ]; + for (const result of item.results ?? []) { + parts.push({ + type: 'tool_call_response', + id: item.id, + response: this.captureMessageContent ? result : undefined, + } satisfies ToolCallResponsePart); + } + + return { + role: 'assistant', + parts, + } satisfies ChatMessage; + } + + web_search_call(item: ResponseFunctionWebSearch): ChatMessage { + return { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: item.id, + name: item.type, + // @ts-expect-error: action is missing on Responses.ResponseFunctionWebSearch type + arguments: this.captureMessageContent ? item.action : undefined, + } satisfies ToolCallRequestPart, + ], + } satisfies ChatMessage; + } + + computer_call(item: ResponseComputerToolCall): ChatMessage { + return { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: item.id, + name: item.type, + arguments: this.captureMessageContent ? item.action : undefined, + call_id: item.call_id, + } satisfies ToolCallRequestPart, + ], + } satisfies ChatMessage; + } + + computer_call_output(item: ResponseInputItem.ComputerCallOutput): ChatMessage { + return { + role: 'user', + parts: [ + { + type: 'tool_call_response', + id: item.id, + response: this.captureMessageContent ? item.output : undefined, + call_id: item.call_id, + } satisfies ToolCallResponsePart, + ], + } satisfies ChatMessage; + } + + code_interpreter_call(item: ResponseCodeInterpreterToolCall): ChatMessage { + const parts: Array = [ + { + type: 'tool_call', + id: item.id, + name: item.type, + arguments: this.captureMessageContent ? item.code : undefined, + } satisfies ToolCallRequestPart, + ]; + for (const output of item.outputs ?? []) { + switch (output.type) { + case 'image': + parts.push({ + type: 'tool_call_response', + id: item.id, + response: this.captureMessageContent ? output.url : undefined, + } satisfies ToolCallResponsePart); + break; + case 'logs': + parts.push({ + type: 'tool_call_response', + id: item.id, + response: this.captureMessageContent ? output.logs : undefined, + } satisfies ToolCallResponsePart); + break; + } + } + + return { + role: 'assistant', + parts, + } satisfies ChatMessage; + } + + image_generation_call( + item: + | ResponseInputItem.ImageGenerationCall + | ResponseOutputItem.ImageGenerationCall, + ): ChatMessage { + return { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: item.id, + name: item.type, + } satisfies ToolCallRequestPart, + { + type: 'tool_call_response', + id: item.id, + response: this.captureMessageContent ? item.result : undefined, + } satisfies ToolCallResponsePart, + ], + } satisfies ChatMessage; + } + + function_call_output(item: ResponseInputItem.FunctionCallOutput): ChatMessage { + return { + role: 'user', + parts: [ + { + type: 'tool_call_response', + id: item.id, + response: this.captureMessageContent ? item.output : undefined, + call_id: item.call_id, + } satisfies ToolCallResponsePart, + ], + } satisfies ChatMessage; + } + + local_shell_call( + item: + | ResponseInputItem.LocalShellCall + | ResponseOutputItem.LocalShellCall, + ): ChatMessage { + return { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: item.id, + name: item.type, + arguments: this.captureMessageContent ? item.action : undefined, + call_id: item.call_id, + } satisfies ToolCallRequestPart, + ], + } satisfies ChatMessage; + } + + local_shell_call_output(item: ResponseInputItem.LocalShellCallOutput): ChatMessage { + return { + role: 'user', + parts: [ + { + type: 'tool_call_response', + id: item.id, + response: this.captureMessageContent ? item.output : undefined, + } satisfies ToolCallResponsePart, + ], + } satisfies ChatMessage; + } + + mcp_call( + item: ResponseInputItem.McpCall | ResponseOutputItem.McpCall, + ): ChatMessage { + return { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: item.id, + name: item.type, + arguments: this.captureMessageContent ? `${item.name}(${item.arguments})` : undefined, + server: item.server_label, + } satisfies ToolCallRequestPart, + { + type: 'tool_call_response', + id: item.id, + response: item.error + ? item.error + : this.captureMessageContent + ? item.output + : undefined, + server: item.server_label, + } satisfies ToolCallResponsePart, + ], + } satisfies ChatMessage; + } + + mcp_list_tools(item: ResponseInputItem.McpListTools): ChatMessage { + return { + role: 'assistant', + parts: [ + { + type: 'tool_call_response', + id: item.id, + response: item.error + ? item.error + : this.captureMessageContent + ? item.tools + : undefined, + server: item.server_label, + } satisfies ToolCallResponsePart, + ], + } satisfies ChatMessage; + } + + mcp_approval_request(item: ResponseInputItem.McpApprovalRequest): ChatMessage { + return { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: item.id, + name: `${item.type}${this.captureMessageContent ? `: ${item.name}` : ''}`, + arguments: this.captureMessageContent ? item.arguments : undefined, + server: item.server_label, + } satisfies ToolCallRequestPart, + ], + } satisfies ChatMessage; + } + + mcp_approval_response(item: ResponseInputItem.McpApprovalResponse): ChatMessage { + return { + role: 'user', + parts: [ + { + type: 'tool_call_response', + response: this.captureMessageContent ? item.approve : undefined, + } satisfies ToolCallResponsePart, + ], + } satisfies ChatMessage; + } + + custom_tool_call_output(item: ResponseCustomToolCallOutput): ChatMessage { + return { + role: 'user', + parts: [ + { + type: 'tool_call_response', + id: item.id, + response: this.captureMessageContent ? item.output : undefined, + call_id: item.call_id, + } satisfies ToolCallResponsePart, + ], + } satisfies ChatMessage; + } + + item_reference(item: ResponseInputItem.ItemReference): ChatMessage { + return { + role: 'assistant', + parts: [ + { + type: 'item_reference', + id: item.id, + } satisfies GenericPart, + ], + } satisfies ChatMessage; + } +} + +class ConvertResponseOutputsToOutputMessagesUseCase { + constructor(private readonly captureMessageContent = false) { } + + convert(responseOutput: Array): OutputMessages { + const parts: Array = responseOutput.flatMap((item: ResponseOutputItem) => this[item.type](item as never)); + + return [ + { + role: 'assistant', + parts, + finish_reason: parts[parts.length - 1]?.type === 'tool_call' ? 'tool_call' : 'stop', + }, + ]; + + } + + message(item: ResponseOutputMessage): Array { + const parts: Array = []; + for (const content of item.content) { + switch (content.type) { + case 'output_text': + if (this.captureMessageContent) { + parts.push({ + type: 'text', + content: content.text, + } satisfies TextPart); + } else { + parts.push({ + type: 'text', + content: undefined, + } satisfies GenericPart); + } + break; + case 'refusal': + parts.push({ + type: content.type, + refusal: content.refusal, + } satisfies GenericPart); + break; + } + } + + return parts; + } + + function_call(item: ResponseFunctionToolCall): Array { + return [ + { + type: 'tool_call', + id: item.id, + name: item.name, + arguments: this.captureMessageContent ? item.arguments : undefined, + call_id: item.call_id, + } satisfies ToolCallRequestPart + ]; + } + + custom_tool_call(item: ResponseCustomToolCall): Array { + return [ + { + type: 'tool_call', + id: item.id, + name: item.name, + arguments: this.captureMessageContent ? item.input : undefined, + call_id: item.call_id, + } satisfies ToolCallRequestPart + ]; + } + + reasoning(item: ResponseReasoningItem): Array { + const parts: Array = []; + for (const summary of item.summary) { + parts.push({ + type: item.type, + text: this.captureMessageContent ? summary.text : undefined, + }); + } + if (item.content) { + for (const content of item.content) { + parts.push({ + type: item.type, + text: this.captureMessageContent ? content.text : undefined, + }); + } + } + + return parts; + } + + file_search_call(item: ResponseFileSearchToolCall): Array { + const parts: Array = [ + { + type: 'tool_call', + id: item.id, + name: item.type, + arguments: this.captureMessageContent ? item.queries : undefined, + } satisfies ToolCallRequestPart + ]; + for (const result of item.results ?? []) { + parts.push({ + type: 'tool_call_response', + id: item.id, + response: this.captureMessageContent ? result : undefined, + } satisfies ToolCallResponsePart); + } + + return parts; + } + + web_search_call(item: ResponseFunctionWebSearch): Array { + return [ + { + type: 'tool_call', + id: item.id, + name: item.type, + // @ts-expect-error: action is missing on Responses.ResponseFunctionWebSearch type + arguments: this.captureMessageContent ? item.action : undefined, + } satisfies ToolCallRequestPart, + ]; + } + + computer_call(item: ResponseComputerToolCall): Array { + return [ + { + type: 'tool_call', + id: item.id, + name: item.type, + arguments: this.captureMessageContent ? item.action : undefined, + call_id: item.call_id, + } satisfies ToolCallRequestPart, + ]; + } + + code_interpreter_call(item: ResponseCodeInterpreterToolCall): Array { + const parts: Array = [ + { + type: 'tool_call', + id: item.id, + name: item.type, + arguments: this.captureMessageContent ? item.code : undefined, + } satisfies ToolCallRequestPart, + ]; + for (const output of item.outputs ?? []) { + switch (output.type) { + case 'image': + parts.push({ + type: 'tool_call_response', + id: item.id, + response: this.captureMessageContent ? output.url : undefined, + } satisfies ToolCallResponsePart); + break; + case 'logs': + parts.push({ + type: 'tool_call_response', + id: item.id, + response: this.captureMessageContent ? output.logs : undefined, + } satisfies ToolCallResponsePart); + break; + } + } + + return parts; + } + + image_generation_call( + item: ResponseOutputItem.ImageGenerationCall, + ): Array { + return [ + { + type: 'tool_call', + id: item.id, + name: item.type, + } satisfies ToolCallRequestPart, + { + type: 'tool_call_response', + id: item.id, + response: this.captureMessageContent ? item.result : undefined, + } satisfies ToolCallResponsePart, + ]; + } + + local_shell_call(item: ResponseOutputItem.LocalShellCall): Array { + return [ + { + type: 'tool_call', + id: item.id, + name: item.type, + arguments: this.captureMessageContent ? item.action : undefined, + call_id: item.call_id, + } satisfies ToolCallRequestPart, + ]; + } + + mcp_call(item: ResponseOutputItem.McpCall): Array { + return [ + { + type: 'tool_call', + id: item.id, + name: item.name, + arguments: this.captureMessageContent ? `${item.name}(${item.arguments})` : undefined, + server: item.server_label, + } satisfies ToolCallRequestPart, + { + type: 'tool_call_response', + id: item.id, + response: item.error + ? item.error + : this.captureMessageContent + ? item.output + : undefined, + server: item.server_label, + } satisfies ToolCallResponsePart + ]; + } + + mcp_list_tools(item: ResponseOutputItem.McpListTools): Array { + return [ + { + type: 'tool_call_response', + id: item.id, + response: item.error + ? item.error + : this.captureMessageContent + ? item.tools + : undefined, + server: item.server_label, + } satisfies ToolCallResponsePart, + ]; + } + + mcp_approval_request( + item: ResponseOutputItem.McpApprovalRequest, + ): Array { + return [ + { + type: 'tool_call', + id: item.id, + name: `${item.type}${this.captureMessageContent ? `: ${item.name}` : ''}`, + arguments: this.captureMessageContent ? item.arguments : undefined, + server: item.server_label, + } satisfies ToolCallRequestPart, + ]; + } } function isTextContent( @@ -853,12 +1832,13 @@ function isTextContent( return value.type === 'text'; } -function isStreamPromise( - params: ChatCompletionCreateParams | undefined, - value: APIPromise | ChatCompletion> -): value is APIPromise> { - if (params && params.stream) { - return true; - } - return false; +function isStreamPromise< + Params extends { stream?: boolean | null } | undefined, + Chunk, + NonStream +>( + params: Params, + value: APIPromise | NonStream> +): value is APIPromise> { + return Boolean(params?.stream); } diff --git a/packages/instrumentation-openai/src/internal-types.ts b/packages/instrumentation-openai/src/internal-types.ts index c1601ddfac..e4511b6572 100644 --- a/packages/instrumentation-openai/src/internal-types.ts +++ b/packages/instrumentation-openai/src/internal-types.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { AnyValue } from '@opentelemetry/api-logs'; +import type { AnyValue } from '@opentelemetry/api-logs'; // This mimicks `APIPromise` from `openai` sufficiently for usage in this // instrumentation. OpenAI's APIPromise adds some methods, but we don't use @@ -22,13 +22,13 @@ import { AnyValue } from '@opentelemetry/api-logs'; export type APIPromise = Promise; export type GenAIFunction = { - name: string; + name: string | undefined; arguments?: AnyValue; }; export type GenAIToolCall = { id: string; - type: string; + type: string | undefined; function?: GenAIFunction; }; @@ -65,3 +65,111 @@ export type GenAIToolMessageEventBody = { content?: AnyValue; id: string; }; + + +// Modelling output messages JSON schema: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-output-messages.json + +/** + * Represents the reason for finishing the generation. + */ +export type FinishReason = + | 'stop' + | 'length' + | 'content_filter' + | 'tool_call' + | 'error'; + +/** + * Role of the entity that created the message. + */ +export type Role = 'system' | 'user' | 'assistant' | 'tool'; + +/** + * Represents an arbitrary message part with any type and properties. + * This allows for extensibility with custom message part types. + */ +export interface GenericPart { + /** The type of the content captured in this part. */ + type: string; + [key: string]: unknown; +} + +/** + * Represents text content sent to or received from the model. + */ +export interface TextPart extends GenericPart { + /** The type of the content captured in this part. */ + type: 'text'; + /** Text content sent to or received from the model. */ + content: string; +} + +/** + * Represents a tool call requested by the model. + */ +export interface ToolCallRequestPart extends GenericPart { + /** The type of the content captured in this part. */ + type: 'tool_call'; + /** Unique identifier for the tool call. */ + id?: string | null; + /** Name of the tool. */ + name: string; + /** Arguments for the tool call. */ + arguments?: unknown | null; +} + +/** + * Represents a tool call result sent to the model or a built-in tool call outcome and details. + */ +export interface ToolCallResponsePart extends GenericPart { + /** The type of the content captured in this part. */ + type: 'tool_call_response'; + /** Unique tool call identifier. */ + id?: string | null; + /** Tool call response. */ + response: unknown; +} + +/** + * Union of all possible message parts that can make up the message content. + */ +export type MessagePart = + | TextPart + | ToolCallRequestPart + | ToolCallResponsePart + | GenericPart; + +/** + * Represents a chat message sent to or from the model. + */ +export interface ChatMessage { + /** Role of the entity that created the message. */ + role: Role | string; + /** List of message parts that make up the message content. */ + parts: MessagePart[]; + [key: string]: unknown; +} + +/** + * Represents the list of input messages sent to the model. + */ +export type InputMessages = ChatMessage[]; + +/** + * Represents an output message generated by the model or agent. + * The output message captures a specific response (choice, candidate). + */ +export interface OutputMessage { + /** Role of the entity that created the message. */ + role: Role | string; + /** List of message parts that make up the message content. */ + parts: MessagePart[]; + /** Reason for finishing the generation. */ + finish_reason: FinishReason | string; + [key: string]: unknown; +} + +/** + * Represents the list of output messages generated by the model or agent. + */ +export type OutputMessages = OutputMessage[]; diff --git a/packages/instrumentation-openai/src/semconv.ts b/packages/instrumentation-openai/src/semconv.ts index 36591a0c74..be0dce7e85 100644 --- a/packages/instrumentation-openai/src/semconv.ts +++ b/packages/instrumentation-openai/src/semconv.ts @@ -32,6 +32,220 @@ */ export const ATTR_EVENT_NAME = 'event.name' as const; +/** + * Free-form description of the GenAI agent provided by the application. + * + * @example Helps with math problems + * @example Generates fiction stories + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const ATTR_GEN_AI_AGENT_DESCRIPTION = 'gen_ai.agent.description' as const; + +/** + * The unique identifier of the GenAI agent. + * + * @example asst_5j66UpCpwteGg4YSxUnt7lPY + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const ATTR_GEN_AI_AGENT_ID = 'gen_ai.agent.id' as const; + +/** + * Human-readable name of the GenAI agent provided by the application. + * + * @example Math Tutor + * @example Fiction Writer + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const ATTR_GEN_AI_AGENT_NAME = 'gen_ai.agent.name' as const; + +/** + * Deprecated, use Event API to report completions contents. + * + * @example [{'role': 'assistant', 'content': 'The capital of France is Paris.'}] + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Removed, no replacement at this time. + */ +export const ATTR_GEN_AI_COMPLETION = 'gen_ai.completion' as const; + +/** + * The unique identifier for a conversation (session, thread), used to store and correlate messages within this conversation. + * + * @example conv_5j66UpCpwteGg4YSxUnt7lPY + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const ATTR_GEN_AI_CONVERSATION_ID = 'gen_ai.conversation.id' as const; + +/** + * The data source identifier. + * + * @example H7STPQYOND + * + * @note Data sources are used by AI agents and RAG applications to store grounding data. A data source may be an external database, object store, document collection, website, or any other storage system used by the GenAI agent or application. The `gen_ai.data_source.id` **SHOULD** match the identifier used by the GenAI system rather than a name specific to the external storage, such as a database or object store. Semantic conventions referencing `gen_ai.data_source.id` **MAY** also leverage additional attributes, such as `db.*`, to further identify and describe the data source. + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const ATTR_GEN_AI_DATA_SOURCE_ID = 'gen_ai.data_source.id' as const; + +/** + * The chat history provided to the model as an input. + * + * @example [ + * { + * "role": "user", + * "parts": [ + * { + * "type": "text", + * "content": "Weather in Paris?" + * } + * ] + * }, + * { + * "role": "assistant", + * "parts": [ + * { + * "type": "tool_call", + * "id": "call_VSPygqKTWdrhaFErNvMV18Yl", + * "name": "get_weather", + * "arguments": { + * "location": "Paris" + * } + * } + * ] + * }, + * { + * "role": "tool", + * "parts": [ + * { + * "type": "tool_call_response", + * "id": " call_VSPygqKTWdrhaFErNvMV18Yl", + * "result": "rainy, 57°F" + * } + * ] + * } + * ] + * + * @note Instrumentations **MUST** follow [Input messages JSON schema](/docs/gen-ai/gen-ai-input-messages.json). + * When the attribute is recorded on events, it **MUST** be recorded in structured + * form. When recorded on spans, it **MAY** be recorded as a JSON string if structured + * format is not supported and **SHOULD** be recorded in structured form otherwise. + * + * Messages **MUST** be provided in the order they were sent to the model. + * Instrumentations **MAY** provide a way for users to filter or truncate + * input messages. + * + * > [!Warning] + * > This attribute is likely to contain sensitive information including user/PII data. + * + * See [Recording content on attributes](/docs/gen-ai/gen-ai-spans.md#recording-content-on-attributes) + * section for more details. + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const ATTR_GEN_AI_INPUT_MESSAGES = 'gen_ai.input.messages' as const; + +/** + * Deprecated, use `gen_ai.output.type`. + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Replaced by `gen_ai.output.type`. + */ +export const ATTR_GEN_AI_OPENAI_REQUEST_RESPONSE_FORMAT = 'gen_ai.openai.request.response_format' as const; + +/** + * Enum value "json_object" for attribute {@link ATTR_GEN_AI_OPENAI_REQUEST_RESPONSE_FORMAT}. + * + * JSON object response format + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_OPENAI_REQUEST_RESPONSE_FORMAT_VALUE_JSON_OBJECT = 'json_object' as const; + +/** + * Enum value "json_schema" for attribute {@link ATTR_GEN_AI_OPENAI_REQUEST_RESPONSE_FORMAT}. + * + * JSON schema response format + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_OPENAI_REQUEST_RESPONSE_FORMAT_VALUE_JSON_SCHEMA = 'json_schema' as const; + +/** + * Enum value "text" for attribute {@link ATTR_GEN_AI_OPENAI_REQUEST_RESPONSE_FORMAT}. + * + * Text response format + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_OPENAI_REQUEST_RESPONSE_FORMAT_VALUE_TEXT = 'text' as const; + +/** + * Deprecated, use `gen_ai.request.seed`. + * + * @example 100 + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Replaced by `gen_ai.request.seed`. + */ +export const ATTR_GEN_AI_OPENAI_REQUEST_SEED = 'gen_ai.openai.request.seed' as const; + +/** + * Deprecated, use `openai.request.service_tier`. + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Replaced by `openai.request.service_tier`. + */ +export const ATTR_GEN_AI_OPENAI_REQUEST_SERVICE_TIER = 'gen_ai.openai.request.service_tier' as const; + +/** + * Enum value "auto" for attribute {@link ATTR_GEN_AI_OPENAI_REQUEST_SERVICE_TIER}. + * + * The system will utilize scale tier credits until they are exhausted. + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_OPENAI_REQUEST_SERVICE_TIER_VALUE_AUTO = 'auto' as const; + +/** + * Enum value "default" for attribute {@link ATTR_GEN_AI_OPENAI_REQUEST_SERVICE_TIER}. + * + * The system will utilize the default scale tier. + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_OPENAI_REQUEST_SERVICE_TIER_VALUE_DEFAULT = 'default' as const; + +/** + * Deprecated, use `openai.response.service_tier`. + * + * @example scale + * @example default + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Replaced by `openai.response.service_tier`. + */ +export const ATTR_GEN_AI_OPENAI_RESPONSE_SERVICE_TIER = 'gen_ai.openai.response.service_tier' as const; + +/** + * Deprecated, use `openai.response.system_fingerprint`. + * + * @example fp_44709d6fcb + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Replaced by `openai.response.system_fingerprint`. + */ +export const ATTR_GEN_AI_OPENAI_RESPONSE_SYSTEM_FINGERPRINT = 'gen_ai.openai.response.system_fingerprint' as const; + /** * The name of the operation being performed. * @@ -41,6 +255,337 @@ export const ATTR_EVENT_NAME = 'event.name' as const; */ export const ATTR_GEN_AI_OPERATION_NAME = 'gen_ai.operation.name' as const; +/** + * Enum value "chat" for attribute {@link ATTR_GEN_AI_OPERATION_NAME}. + * + * Chat completion operation such as [OpenAI Chat API](https://platform.openai.com/docs/api-reference/chat) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_OPERATION_NAME_VALUE_CHAT = 'chat' as const; + +/** + * Enum value "create_agent" for attribute {@link ATTR_GEN_AI_OPERATION_NAME}. + * + * Create GenAI agent + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_OPERATION_NAME_VALUE_CREATE_AGENT = 'create_agent' as const; + +/** + * Enum value "embeddings" for attribute {@link ATTR_GEN_AI_OPERATION_NAME}. + * + * Embeddings operation such as [OpenAI Create embeddings API](https://platform.openai.com/docs/api-reference/embeddings/create) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_OPERATION_NAME_VALUE_EMBEDDINGS = 'embeddings' as const; + +/** + * Enum value "execute_tool" for attribute {@link ATTR_GEN_AI_OPERATION_NAME}. + * + * Execute a tool + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL = 'execute_tool' as const; + +/** + * Enum value "generate_content" for attribute {@link ATTR_GEN_AI_OPERATION_NAME}. + * + * Multimodal content generation operation such as [Gemini Generate Content](https://ai.google.dev/api/generate-content) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_OPERATION_NAME_VALUE_GENERATE_CONTENT = 'generate_content' as const; + +/** + * Enum value "invoke_agent" for attribute {@link ATTR_GEN_AI_OPERATION_NAME}. + * + * Invoke GenAI agent + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT = 'invoke_agent' as const; + +/** + * Enum value "text_completion" for attribute {@link ATTR_GEN_AI_OPERATION_NAME}. + * + * Text completions operation such as [OpenAI Completions API (Legacy)](https://platform.openai.com/docs/api-reference/completions) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_OPERATION_NAME_VALUE_TEXT_COMPLETION = 'text_completion' as const; + +/** + * Messages returned by the model where each message represents a specific model response (choice, candidate). + * + * @example [ + * { + * "role": "assistant", + * "parts": [ + * { + * "type": "text", + * "content": "The weather in Paris is currently rainy with a temperature of 57°F." + * } + * ], + * "finish_reason": "stop" + * } + * ] + * + * @note Instrumentations **MUST** follow [Output messages JSON schema](/docs/gen-ai/gen-ai-output-messages.json) + * + * Each message represents a single output choice/candidate generated by + * the model. Each message corresponds to exactly one generation + * (choice/candidate) and vice versa - one choice cannot be split across + * multiple messages or one message cannot contain parts from multiple choices. + * + * When the attribute is recorded on events, it **MUST** be recorded in structured + * form. When recorded on spans, it **MAY** be recorded as a JSON string if structured + * format is not supported and **SHOULD** be recorded in structured form otherwise. + * + * Instrumentations **MAY** provide a way for users to filter or truncate + * output messages. + * + * > [!Warning] + * > This attribute is likely to contain sensitive information including user/PII data. + * + * See [Recording content on attributes](/docs/gen-ai/gen-ai-spans.md#recording-content-on-attributes) + * section for more details. + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const ATTR_GEN_AI_OUTPUT_MESSAGES = 'gen_ai.output.messages' as const; + +/** + * Represents the content type requested by the client. + * + * @note This attribute **SHOULD** be used when the client requests output of a specific type. The model may return zero or more outputs of this type. + * This attribute specifies the output modality and not the actual output format. For example, if an image is requested, the actual output could be a URL pointing to an image file. + * Additional output format details may be recorded in the future in the `gen_ai.output.{type}.*` attributes. + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const ATTR_GEN_AI_OUTPUT_TYPE = 'gen_ai.output.type' as const; + +/** + * Enum value "image" for attribute {@link ATTR_GEN_AI_OUTPUT_TYPE}. + * + * Image + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_OUTPUT_TYPE_VALUE_IMAGE = 'image' as const; + +/** + * Enum value "json" for attribute {@link ATTR_GEN_AI_OUTPUT_TYPE}. + * + * JSON object with known or unknown schema + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_OUTPUT_TYPE_VALUE_JSON = 'json' as const; + +/** + * Enum value "speech" for attribute {@link ATTR_GEN_AI_OUTPUT_TYPE}. + * + * Speech + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_OUTPUT_TYPE_VALUE_SPEECH = 'speech' as const; + +/** + * Enum value "text" for attribute {@link ATTR_GEN_AI_OUTPUT_TYPE}. + * + * Plain text + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_OUTPUT_TYPE_VALUE_TEXT = 'text' as const; + +/** + * Deprecated, use Event API to report prompt contents. + * + * @example [{'role': 'user', 'content': 'What is the capital of France?'}] + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Removed, no replacement at this time. + */ +export const ATTR_GEN_AI_PROMPT = 'gen_ai.prompt' as const; + +/** + * The Generative AI provider as identified by the client or server instrumentation. + * + * @note The attribute **SHOULD** be set based on the instrumentation's best + * knowledge and may differ from the actual model provider. + * + * Multiple providers, including Azure OpenAI, Gemini, and AI hosting platforms + * are accessible using the OpenAI REST API and corresponding client libraries, + * but may proxy or host models from different providers. + * + * The `gen_ai.request.model`, `gen_ai.response.model`, and `server.address` + * attributes may help identify the actual system in use. + * + * The `gen_ai.provider.name` attribute acts as a discriminator that + * identifies the GenAI telemetry format flavor specific to that provider + * within GenAI semantic conventions. + * It **SHOULD** be set consistently with provider-specific attributes and signals. + * For example, GenAI spans, metrics, and events related to AWS Bedrock + * should have the `gen_ai.provider.name` set to `aws.bedrock` and include + * applicable `aws.bedrock.*` attributes and are not expected to include + * `openai.*` attributes. + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const ATTR_GEN_AI_PROVIDER_NAME = 'gen_ai.provider.name' as const; + +/** + * Enum value "anthropic" for attribute {@link ATTR_GEN_AI_PROVIDER_NAME}. + * + * [Anthropic](https://www.anthropic.com/) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_PROVIDER_NAME_VALUE_ANTHROPIC = 'anthropic' as const; + +/** + * Enum value "aws.bedrock" for attribute {@link ATTR_GEN_AI_PROVIDER_NAME}. + * + * [AWS Bedrock](https://aws.amazon.com/bedrock) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_PROVIDER_NAME_VALUE_AWS_BEDROCK = 'aws.bedrock' as const; + +/** + * Enum value "azure.ai.inference" for attribute {@link ATTR_GEN_AI_PROVIDER_NAME}. + * + * Azure AI Inference + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_INFERENCE = 'azure.ai.inference' as const; + +/** + * Enum value "azure.ai.openai" for attribute {@link ATTR_GEN_AI_PROVIDER_NAME}. + * + * [Azure OpenAI](https://azure.microsoft.com/products/ai-services/openai-service/) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_PROVIDER_NAME_VALUE_AZURE_AI_OPENAI = 'azure.ai.openai' as const; + +/** + * Enum value "cohere" for attribute {@link ATTR_GEN_AI_PROVIDER_NAME}. + * + * [Cohere](https://cohere.com/) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_PROVIDER_NAME_VALUE_COHERE = 'cohere' as const; + +/** + * Enum value "deepseek" for attribute {@link ATTR_GEN_AI_PROVIDER_NAME}. + * + * [DeepSeek](https://www.deepseek.com/) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_PROVIDER_NAME_VALUE_DEEPSEEK = 'deepseek' as const; + +/** + * Enum value "gcp.gemini" for attribute {@link ATTR_GEN_AI_PROVIDER_NAME}. + * + * [Gemini](https://cloud.google.com/products/gemini) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_PROVIDER_NAME_VALUE_GCP_GEMINI = 'gcp.gemini' as const; + +/** + * Enum value "gcp.gen_ai" for attribute {@link ATTR_GEN_AI_PROVIDER_NAME}. + * + * Any Google generative AI endpoint + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_PROVIDER_NAME_VALUE_GCP_GEN_AI = 'gcp.gen_ai' as const; + +/** + * Enum value "gcp.vertex_ai" for attribute {@link ATTR_GEN_AI_PROVIDER_NAME}. + * + * [Vertex AI](https://cloud.google.com/vertex-ai) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_PROVIDER_NAME_VALUE_GCP_VERTEX_AI = 'gcp.vertex_ai' as const; + +/** + * Enum value "groq" for attribute {@link ATTR_GEN_AI_PROVIDER_NAME}. + * + * [Groq](https://groq.com/) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_PROVIDER_NAME_VALUE_GROQ = 'groq' as const; + +/** + * Enum value "ibm.watsonx.ai" for attribute {@link ATTR_GEN_AI_PROVIDER_NAME}. + * + * [IBM Watsonx AI](https://www.ibm.com/products/watsonx-ai) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_PROVIDER_NAME_VALUE_IBM_WATSONX_AI = 'ibm.watsonx.ai' as const; + +/** + * Enum value "mistral_ai" for attribute {@link ATTR_GEN_AI_PROVIDER_NAME}. + * + * [Mistral AI](https://mistral.ai/) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_PROVIDER_NAME_VALUE_MISTRAL_AI = 'mistral_ai' as const; + +/** + * Enum value "openai" for attribute {@link ATTR_GEN_AI_PROVIDER_NAME}. + * + * [OpenAI](https://openai.com/) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_PROVIDER_NAME_VALUE_OPENAI = 'openai' as const; + +/** + * Enum value "perplexity" for attribute {@link ATTR_GEN_AI_PROVIDER_NAME}. + * + * [Perplexity](https://www.perplexity.ai/) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_PROVIDER_NAME_VALUE_PERPLEXITY = 'perplexity' as const; + +/** + * Enum value "x_ai" for attribute {@link ATTR_GEN_AI_PROVIDER_NAME}. + * + * [xAI](https://x.ai/) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_PROVIDER_NAME_VALUE_X_AI = 'x_ai' as const; + +/** + * The target number of candidate completions to return. + * + * @example 3 + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const ATTR_GEN_AI_REQUEST_CHOICE_COUNT = 'gen_ai.request.choice.count' as const; + /** * The encoding formats requested in an embeddings operation, if specified. * @@ -51,8 +596,7 @@ export const ATTR_GEN_AI_OPERATION_NAME = 'gen_ai.operation.name' as const; * * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. */ -export const ATTR_GEN_AI_REQUEST_ENCODING_FORMATS = - 'gen_ai.request.encoding_formats' as const; +export const ATTR_GEN_AI_REQUEST_ENCODING_FORMATS = 'gen_ai.request.encoding_formats' as const; /** * The frequency penalty setting for the GenAI request. @@ -61,8 +605,7 @@ export const ATTR_GEN_AI_REQUEST_ENCODING_FORMATS = * * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. */ -export const ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY = - 'gen_ai.request.frequency_penalty' as const; +export const ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY = 'gen_ai.request.frequency_penalty' as const; /** * The maximum number of tokens the model generates for a request. @@ -71,8 +614,7 @@ export const ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY = * * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. */ -export const ATTR_GEN_AI_REQUEST_MAX_TOKENS = - 'gen_ai.request.max_tokens' as const; +export const ATTR_GEN_AI_REQUEST_MAX_TOKENS = 'gen_ai.request.max_tokens' as const; /** * The name of the GenAI model a request is being made to. @@ -90,8 +632,16 @@ export const ATTR_GEN_AI_REQUEST_MODEL = 'gen_ai.request.model' as const; * * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. */ -export const ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY = - 'gen_ai.request.presence_penalty' as const; +export const ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY = 'gen_ai.request.presence_penalty' as const; + +/** + * Requests with same seed value more likely to return same result. + * + * @example 100 + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const ATTR_GEN_AI_REQUEST_SEED = 'gen_ai.request.seed' as const; /** * List of sequences that the model will use to stop generating further tokens. @@ -100,8 +650,7 @@ export const ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY = * * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. */ -export const ATTR_GEN_AI_REQUEST_STOP_SEQUENCES = - 'gen_ai.request.stop_sequences' as const; +export const ATTR_GEN_AI_REQUEST_STOP_SEQUENCES = 'gen_ai.request.stop_sequences' as const; /** * The temperature setting for the GenAI request. @@ -110,8 +659,16 @@ export const ATTR_GEN_AI_REQUEST_STOP_SEQUENCES = * * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. */ -export const ATTR_GEN_AI_REQUEST_TEMPERATURE = - 'gen_ai.request.temperature' as const; +export const ATTR_GEN_AI_REQUEST_TEMPERATURE = 'gen_ai.request.temperature' as const; + +/** + * The top_k sampling setting for the GenAI request. + * + * @example 1.0 + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const ATTR_GEN_AI_REQUEST_TOP_K = 'gen_ai.request.top_k' as const; /** * The top_p sampling setting for the GenAI request. @@ -130,8 +687,7 @@ export const ATTR_GEN_AI_REQUEST_TOP_P = 'gen_ai.request.top_p' as const; * * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. */ -export const ATTR_GEN_AI_RESPONSE_FINISH_REASONS = - 'gen_ai.response.finish_reasons' as const; +export const ATTR_GEN_AI_RESPONSE_FINISH_REASONS = 'gen_ai.response.finish_reasons' as const; /** * The unique identifier for the completion. @@ -152,25 +708,236 @@ export const ATTR_GEN_AI_RESPONSE_ID = 'gen_ai.response.id' as const; export const ATTR_GEN_AI_RESPONSE_MODEL = 'gen_ai.response.model' as const; /** - * The Generative AI product as identified by the client or server instrumentation. + * Deprecated, use `gen_ai.provider.name` instead. + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Replaced by `gen_ai.provider.name`. + */ +export const ATTR_GEN_AI_SYSTEM = 'gen_ai.system' as const; + +/** + * Enum value "anthropic" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * Anthropic + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_SYSTEM_VALUE_ANTHROPIC = 'anthropic' as const; + +/** + * Enum value "aws.bedrock" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * AWS Bedrock + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_SYSTEM_VALUE_AWS_BEDROCK = 'aws.bedrock' as const; + +/** + * Enum value "az.ai.inference" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * Azure AI Inference + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_SYSTEM_VALUE_AZ_AI_INFERENCE = 'az.ai.inference' as const; + +/** + * Enum value "az.ai.openai" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * Azure OpenAI + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_SYSTEM_VALUE_AZ_AI_OPENAI = 'az.ai.openai' as const; + +/** + * Enum value "azure.ai.inference" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * Azure AI Inference + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_SYSTEM_VALUE_AZURE_AI_INFERENCE = 'azure.ai.inference' as const; + +/** + * Enum value "azure.ai.openai" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * Azure OpenAI + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_SYSTEM_VALUE_AZURE_AI_OPENAI = 'azure.ai.openai' as const; + +/** + * Enum value "cohere" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * Cohere + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_SYSTEM_VALUE_COHERE = 'cohere' as const; + +/** + * Enum value "deepseek" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * DeepSeek + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_SYSTEM_VALUE_DEEPSEEK = 'deepseek' as const; + +/** + * Enum value "gcp.gemini" for attribute {@link ATTR_GEN_AI_SYSTEM}. * - * @example "openai" + * Gemini * - * @note The `gen_ai.system` describes a family of GenAI models with specific model identified - * by `gen_ai.request.model` and `gen_ai.response.model` attributes. + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_SYSTEM_VALUE_GCP_GEMINI = 'gcp.gemini' as const; + +/** + * Enum value "gcp.gen_ai" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * Any Google generative AI endpoint * - * The actual GenAI product may differ from the one identified by the client. - * Multiple systems, including Azure OpenAI and Gemini, are accessible by OpenAI client - * libraries. In such cases, the `gen_ai.system` is set to `openai` based on the - * instrumentation's best knowledge, instead of the actual system. The `server.address` - * attribute may help identify the actual system in use for `openai`. + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_SYSTEM_VALUE_GCP_GEN_AI = 'gcp.gen_ai' as const; + +/** + * Enum value "gcp.vertex_ai" for attribute {@link ATTR_GEN_AI_SYSTEM}. * - * For custom model, a custom friendly name **SHOULD** be used. - * If none of these options apply, the `gen_ai.system` **SHOULD** be set to `_OTHER`. + * Vertex AI + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_SYSTEM_VALUE_GCP_VERTEX_AI = 'gcp.vertex_ai' as const; + +/** + * Enum value "gemini" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * Gemini + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Replaced by `gcp.gemini`. + */ +export const GEN_AI_SYSTEM_VALUE_GEMINI = 'gemini' as const; + +/** + * Enum value "groq" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * Groq + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_SYSTEM_VALUE_GROQ = 'groq' as const; + +/** + * Enum value "ibm.watsonx.ai" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * IBM Watsonx AI + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_SYSTEM_VALUE_IBM_WATSONX_AI = 'ibm.watsonx.ai' as const; + +/** + * Enum value "mistral_ai" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * Mistral AI + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_SYSTEM_VALUE_MISTRAL_AI = 'mistral_ai' as const; + +/** + * Enum value "openai" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * OpenAI + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_SYSTEM_VALUE_OPENAI = 'openai' as const; + +/** + * Enum value "perplexity" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * Perplexity + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_SYSTEM_VALUE_PERPLEXITY = 'perplexity' as const; + +/** + * Enum value "vertex_ai" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * Vertex AI + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Replaced by `gcp.vertex_ai`. + */ +export const GEN_AI_SYSTEM_VALUE_VERTEX_AI = 'vertex_ai' as const; + +/** + * Enum value "xai" for attribute {@link ATTR_GEN_AI_SYSTEM}. + * + * xAI + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Replaced by `x_ai`. + */ +export const GEN_AI_SYSTEM_VALUE_XAI = 'xai' as const; + +/** + * The system message or instructions provided to the GenAI model separately from the chat history. + * + * @example [ + * { + * "type": "text", + * "content": "You are an Agent that greet users, always use greetings tool to respond" + * } + * ] + * + * @example [ + * { + * "type": "text", + * "content": "You are a language translator." + * }, + * { + * "type": "text", + * "content": "Your mission is to translate text in English to French." + * } + * ] + * + * @note This attribute **SHOULD** be used when the corresponding provider or API + * allows to provide system instructions or messages separately from the + * chat history. + * + * Instructions that are part of the chat history **SHOULD** be recorded in + * `gen_ai.input.messages` attribute instead. + * + * Instrumentations **MUST** follow [System instructions JSON schema](/docs/gen-ai/gen-ai-system-instructions.json). + * + * When recorded on spans, it **MAY** be recorded as a JSON string if structured + * format is not supported and **SHOULD** be recorded in structured form otherwise. + * + * Instrumentations **MAY** provide a way for users to filter or truncate + * system instructions. + * + * > [!Warning] + * > This attribute may contain sensitive information. + * + * See [Recording content on attributes](/docs/gen-ai/gen-ai-spans.md#recording-content-on-attributes) + * section for more details. * * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. */ -export const ATTR_GEN_AI_SYSTEM = 'gen_ai.system' as const; +export const ATTR_GEN_AI_SYSTEM_INSTRUCTIONS = 'gen_ai.system_instructions' as const; /** * The type of token being counted. @@ -182,6 +949,90 @@ export const ATTR_GEN_AI_SYSTEM = 'gen_ai.system' as const; */ export const ATTR_GEN_AI_TOKEN_TYPE = 'gen_ai.token.type' as const; +/** + * Enum value "input" for attribute {@link ATTR_GEN_AI_TOKEN_TYPE}. + * + * Input tokens (prompt, input, etc.) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_TOKEN_TYPE_VALUE_INPUT = 'input' as const; + +/** + * Enum value "output" for attribute {@link ATTR_GEN_AI_TOKEN_TYPE}. + * + * Output tokens (completion, response, etc.) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Replaced by `output`. + */ +export const GEN_AI_TOKEN_TYPE_VALUE_COMPLETION = 'output' as const; + +/** + * Enum value "output" for attribute {@link ATTR_GEN_AI_TOKEN_TYPE}. + * + * Output tokens (completion, response, etc.) + * + * @experimental This enum value is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const GEN_AI_TOKEN_TYPE_VALUE_OUTPUT = 'output' as const; + +/** + * The tool call identifier. + * + * @example call_mszuSIzqtI65i1wAUOE8w5H4 + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const ATTR_GEN_AI_TOOL_CALL_ID = 'gen_ai.tool.call.id' as const; + +/** + * The tool description. + * + * @example Multiply two numbers + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const ATTR_GEN_AI_TOOL_DESCRIPTION = 'gen_ai.tool.description' as const; + +/** + * Name of the tool utilized by the agent. + * + * @example Flights + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const ATTR_GEN_AI_TOOL_NAME = 'gen_ai.tool.name' as const; + +/** + * Type of the tool utilized by the agent + * + * @example function + * @example extension + * @example datastore + * + * @note Extension: A tool executed on the agent-side to directly call external APIs, bridging the gap between the agent and real-world systems. + * Agent-side operations involve actions that are performed by the agent on the server or within the agent's controlled environment. + * Function: A tool executed on the client-side, where the agent generates parameters for a predefined function, and the client executes the logic. + * Client-side operations are actions taken on the user's end or within the client application. + * Datastore: A tool used by the agent to access and query structured or unstructured external data for retrieval-augmented tasks or knowledge updates. + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const ATTR_GEN_AI_TOOL_TYPE = 'gen_ai.tool.type' as const; + +/** + * Deprecated, use `gen_ai.usage.output_tokens` instead. + * + * @example 42 + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Replaced by `gen_ai.usage.output_tokens`. + */ +export const ATTR_GEN_AI_USAGE_COMPLETION_TOKENS = 'gen_ai.usage.completion_tokens' as const; + /** * The number of tokens used in the GenAI input (prompt). * @@ -189,8 +1040,7 @@ export const ATTR_GEN_AI_TOKEN_TYPE = 'gen_ai.token.type' as const; * * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. */ -export const ATTR_GEN_AI_USAGE_INPUT_TOKENS = - 'gen_ai.usage.input_tokens' as const; +export const ATTR_GEN_AI_USAGE_INPUT_TOKENS = 'gen_ai.usage.input_tokens' as const; /** * The number of tokens used in the GenAI response (completion). @@ -199,21 +1049,104 @@ export const ATTR_GEN_AI_USAGE_INPUT_TOKENS = * * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. */ -export const ATTR_GEN_AI_USAGE_OUTPUT_TOKENS = - 'gen_ai.usage.output_tokens' as const; +export const ATTR_GEN_AI_USAGE_OUTPUT_TOKENS = 'gen_ai.usage.output_tokens' as const; + +/** + * Deprecated, use `gen_ai.usage.input_tokens` instead. + * + * @example 42 + * + * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Replaced by `gen_ai.usage.input_tokens`. + */ +export const ATTR_GEN_AI_USAGE_PROMPT_TOKENS = 'gen_ai.usage.prompt_tokens' as const; + +/** + * This event describes the assistant message passed to GenAI system. + * + * @experimental This event is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Chat history is reported on `gen_ai.input.messages` attribute on spans or `gen_ai.client.inference.operation.details` event. + */ +export const EVENT_GEN_AI_ASSISTANT_MESSAGE = 'gen_ai.assistant.message' as const; + +/** + * This event describes the Gen AI response message. + * + * @experimental This event is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Chat history is reported on `gen_ai.output.messages` attribute on spans or `gen_ai.client.inference.operation.details` event. + */ +export const EVENT_GEN_AI_CHOICE = 'gen_ai.choice' as const; + +/** + * Describes the details of a GenAI completion request including chat history and parameters. + * + * @note This event is opt-in and could be used to store input and output details independently from traces. + * + * @experimental This event is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS = 'gen_ai.client.inference.operation.details' as const; + +/** + * This event describes the system instructions passed to the GenAI model. + * + * @experimental This event is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Chat history is reported on `gen_ai.system_instructions` attribute on spans or `gen_ai.client.inference.operation.details` event. + */ +export const EVENT_GEN_AI_SYSTEM_MESSAGE = 'gen_ai.system.message' as const; + +/** + * This event describes the response from a tool or function call passed to the GenAI model. + * + * @experimental This event is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Chat history is reported on `gen_ai.input.messages` attribute on spans or `gen_ai.client.inference.operation.details` event. + */ +export const EVENT_GEN_AI_TOOL_MESSAGE = 'gen_ai.tool.message' as const; + +/** + * This event describes the user message passed to the GenAI model. + * + * @experimental This event is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + * + * @deprecated Chat history is reported on `gen_ai.input.messages` attribute on spans or `gen_ai.client.inference.operation.details` event. + */ +export const EVENT_GEN_AI_USER_MESSAGE = 'gen_ai.user.message' as const; + +/** + * GenAI operation duration. + * + * @experimental This metric is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const METRIC_GEN_AI_CLIENT_OPERATION_DURATION = 'gen_ai.client.operation.duration' as const; + +/** + * Number of input and output tokens used. + * + * @experimental This metric is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const METRIC_GEN_AI_CLIENT_TOKEN_USAGE = 'gen_ai.client.token.usage' as const; + +/** + * Generative AI server request duration such as time-to-last byte or last output token. + * + * @experimental This metric is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. + */ +export const METRIC_GEN_AI_SERVER_REQUEST_DURATION = 'gen_ai.server.request.duration' as const; /** - * GenAI operation duration + * Time per output token generated after the first token for successful responses. * * @experimental This metric is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. */ -export const METRIC_GEN_AI_CLIENT_OPERATION_DURATION = - 'gen_ai.client.operation.duration' as const; +export const METRIC_GEN_AI_SERVER_TIME_PER_OUTPUT_TOKEN = 'gen_ai.server.time_per_output_token' as const; /** - * Measures number of input and output tokens used + * Time to generate first token for successful responses. * * @experimental This metric is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`. */ -export const METRIC_GEN_AI_CLIENT_TOKEN_USAGE = - 'gen_ai.client.token.usage' as const; +export const METRIC_GEN_AI_SERVER_TIME_TO_FIRST_TOKEN = 'gen_ai.server.time_to_first_token' as const; diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-adds-genai-conventions.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-adds-genai-conventions.json new file mode 100644 index 0000000000..e6646ed895 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-adds-genai-conventions.json @@ -0,0 +1,73 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "Answer in up to 3 words: Which ocean contains Bouvet Island?" + }, + "status": 200, + "response": { + "id": "resp_67ccd2bed1ec8190b14f964abc0542670bb6a6b452d3795b", + "object": "response", + "created_at": 1741476542, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "type": "message", + "id": "msg_67ccd2bf17f0819081ff3bb2cf6508e60bb6a6b452d3795b", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Atlantic Ocean.", + "annotations": [] + } + ] + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 25 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:37 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-records-all-the-client-options.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-all-the-client-options.json new file mode 100644 index 0000000000..3e4207dc73 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-all-the-client-options.json @@ -0,0 +1,76 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "Answer in up to 3 words: Which ocean contains Bouvet Island?", + "max_output_tokens": 100, + "temperature": 1, + "top_p": 1 + }, + "status": 200, + "response": { + "id": "resp_67ccd2bed1ec8190b14f964abc0542670bb6a6b452d3795b", + "object": "response", + "created_at": 1741476542, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "type": "message", + "id": "msg_67ccd2bf17f0819081ff3bb2cf6508e60bb6a6b452d3795b", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Southern Ocean.", + "annotations": [] + } + ] + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 25 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:38 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-records-code-interpreter-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-code-interpreter-calls.json new file mode 100644 index 0000000000..b7d5908239 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-code-interpreter-calls.json @@ -0,0 +1,93 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "tools": [ + { + "type": "code_interpreter", + "container": "cfile_682e0e8a43c88191a7978f477a09bdf5" + } + ], + "instructions": null, + "input": "Plot and analyse the histogram of the RGB channels for the uploaded image." + }, + "status": 200, + "response": { + "id": "resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0", + "object": "response", + "created_at": 1741294021, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "id": "msg_682d514e268c8191a89c38ea318446200f2610a7ec781a4f", + "content": [ + { + "annotations": [ + { + "file_id": "cfile_682d514b2e00819184b9b07e13557f82", + "index": null, + "type": "container_file_citation", + "container_id": "cntr_682d513bb0c48191b10bd4f8b0b3312200e64562acc2e0af", + "end_index": 0, + "filename": "cfile_682d514b2e00819184b9b07e13557f82.png", + "start_index": 0 + } + ], + "text": "Here is the histogram of the RGB channels for the uploaded image. Each curve represents the distribution of pixel intensities for the red, green, and blue channels. Peaks toward the high end of the intensity scale (right-hand side) suggest a lot of brightness and strong warm tones, matching the orange and light background in the image. If you want a different style of histogram (e.g., overall intensity, or quantized color groups), let me know!", + "type": "output_text", + "logprobs": [] + } + ], + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [ + { + "type": "code_interpreter", + "container": "cfile_682e0e8a43c88191a7978f477a09bdf5" + } + ], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 291, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 314 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:40 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-records-computer-use-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-computer-use-calls.json new file mode 100644 index 0000000000..2d37648120 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-computer-use-calls.json @@ -0,0 +1,109 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "computer-use-preview", + "tools": [ + { + "type": "computer_use_preview", + "display_width": 1024, + "display_height": 768, + "environment": "browser" + } + ], + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Check the latest OpenAI news on bing.com." + } + ] + } + ], + "reasoning": { + "summary": "concise" + }, + "truncation": "auto" + }, + "status": 200, + "response": { + "id": "resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0", + "object": "response", + "created_at": 1741294021, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "computer-use-preview-2025-04-14", + "output": [ + { + "type": "reasoning", + "id": "rs_67cc...", + "summary": [ + { + "type": "summary_text", + "text": "Clicking on the browser address bar." + } + ] + }, + { + "type": "computer_call", + "id": "cu_67cc...", + "call_id": "call_zw3...", + "action": { + "type": "click", + "button": "left", + "x": 156, + "y": 50 + }, + "pending_safety_checks": [], + "status": "completed" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [ + { + "type": "computer_use_preview", + "display_width": 1024, + "display_height": 768, + "environment": "browser" + } + ], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 291, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 314 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:40 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-records-custom-tool-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-custom-tool-calls.json new file mode 100644 index 0000000000..7d2e057f6e --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-custom-tool-calls.json @@ -0,0 +1,84 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "Use the code_exec tool to print hello world to the console.", + "tools": [ + { + "type": "custom", + "name": "code_exec", + "description": "Executes arbitrary Python code." + } + ] + }, + "status": 200, + "response": { + "id": "resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0", + "object": "response", + "created_at": 1741294021, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "id": "rs_6890e972fa7c819ca8bc561526b989170694874912ae0ea6", + "type": "reasoning", + "content": [], + "summary": [] + }, + { + "id": "ctc_6890e975e86c819c9338825b3e1994810694874912ae0ea6", + "type": "custom_tool_call", + "status": "completed", + "call_id": "call_aGiFQkRWSWAIsMQ19fKqxUgb", + "input": "print(\"hello world\")", + "name": "code_exec" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [ + { + "type": "custom", + "name": "code_exec", + "description": "Executes arbitrary Python code." + } + ], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 291, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 314 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:40 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-records-file-search-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-file-search-calls.json new file mode 100644 index 0000000000..8a372dd8e2 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-file-search-calls.json @@ -0,0 +1,109 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "What is deep research by OpenAI?", + "tools": [ + { + "type": "file_search", + "vector_store_ids": [""], + "max_num_results": 2 + } + ], + "include": ["file_search_call.results"] + }, + "status": 200, + "response": { + "id": "resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0", + "object": "response", + "created_at": 1741294021, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "type": "file_search_call", + "id": "fs_67c09ccea8c48191ade9367e3ba71515", + "status": "completed", + "queries": ["What is deep research?"], + "results": [ + { + "id": "file-2dtbBZdjtDKS8eqWxqbgDi", + "filename": "deep_research_blog.pdf", + "score": 0.95, + "text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed euismod, nisl eget aliquam aliquet, nunc nisl aliquet nisl, eget aliquam nisl nisl eget nisl. Sed euismod, nisl eget aliquam aliquet, nunc nisl aliquet nisl, eget aliquam nisl nisl eget nisl." + } + ] + }, + { + "id": "msg_67c09cd3091c819185af2be5d13d87de", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Deep research is a sophisticated capability that allows for extensive inquiry and synthesis of information across various domains. It is designed to conduct multi-step research tasks, gather data from multiple online sources, and provide comprehensive reports similar to what a research analyst would produce. This functionality is particularly useful in fields requiring detailed and accurate information...", + "annotations": [ + { + "type": "file_citation", + "index": 992, + "file_id": "file-2dtbBZdjtDKS8eqWxqbgDi", + "filename": "deep_research_blog.pdf" + }, + { + "type": "file_citation", + "index": 1176, + "file_id": "file-2dtbBZdjtDKS8eqWxqbgDi", + "filename": "deep_research_blog.pdf" + } + ] + } + ] + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [ + { + "type": "file_search", + "vector_store_ids": [""] + } + ], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 291, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 314 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:40 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-records-function-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-function-calls.json new file mode 100644 index 0000000000..aaf76241c9 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-function-calls.json @@ -0,0 +1,161 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": [ + { + "role": "system", + "content": "You are a helpful assistant providing weather updates." + }, + { + "role": "user", + "content": "What is the weather in Paris and Bogotá?" + } + ], + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city, e.g. San Francisco" + } + }, + "required": ["location"] + }, + "strict": true + }, + { + "type": "function", + "name": "send_email", + "description": "Send an email", + "parameters": { + "type": "object", + "properties": { + "to": { + "type": "string", + "description": "The email address of the recipient" + }, + "body": { + "type": "string", + "description": "The body of the email" + } + }, + "required": ["to", "body"] + }, + "strict": true + } + ], + "parallel_tool_calls": true + }, + "status": 200, + "response": { + "id": "resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0", + "object": "response", + "created_at": 1741294021, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "fc_12345xyz", + "call_id": "call_12345xyz", + "type": "function_call", + "name": "get_weather", + "arguments": "{\"location\":\"Paris, France\"}" + }, + { + "id": "fc_67890abc", + "call_id": "call_67890abc", + "type": "function_call", + "name": "get_weather", + "arguments": "{\"location\":\"Bogotá, Colombia\"}" + }, + { + "id": "fc_99999def", + "call_id": "call_99999def", + "type": "function_call", + "name": "send_email", + "arguments": "{\"to\":\"bob@email.com\",\"body\":\"Hi bob\"}" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and country, e.g. San Francisco, USA" + } + }, + "required": ["location"] + } + }, + { + "type": "function", + "name": "send_email", + "description": "Send an email", + "parameters": { + "type": "object", + "properties": { + "to": { + "type": "string", + "description": "The email address of the recipient" + }, + "body": { + "type": "string", + "description": "The body of the email" + } + }, + "required": ["to", "body"] + } + } + ], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 291, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 314 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:40 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-records-image-generation-call.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-image-generation-call.json new file mode 100644 index 0000000000..4363e06dc0 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-image-generation-call.json @@ -0,0 +1,76 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": [ + { + "role": "user", + "content": [ + { "type": "input_text", "text": "Now make it look realistic" } + ] + }, + { + "type": "image_generation_call", + "id": "ig_123" + } + ], + "tools": [{ "type": "image_generation" }] + }, + "status": 200, + "response": { + "id": "resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0", + "object": "response", + "created_at": 1741294021, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "id": "ig_124", + "type": "image_generation_call", + "status": "completed", + "revised_prompt": "A gray tabby cat hugging an otter. The otter is wearing an orange scarf. Both animals are cute and friendly, depicted in a warm, heartwarming, but realistic style.", + "result": "..." + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [{ "type": "image_generation" }], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 291, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 314 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:40 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-records-mcp-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-mcp-calls.json new file mode 100644 index 0000000000..759579484a --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-mcp-calls.json @@ -0,0 +1,126 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "tools": [ + { + "type": "mcp", + "server_label": "dmcp", + "server_description": "A Dungeons and Dragons MCP server to assist with dice rolling.", + "server_url": "https://dmcp-server.deno.dev/sse", + "require_approval": "never", + "allowed_tools": ["roll"] + } + ], + "input": [ + { + "role": "user", + "content": "Roll 2d4+1" + }, + { + "id": "mcpl_68a6102a4968819c8177b05584dd627b0679e572a900e618", + "type": "mcp_list_tools", + "server_label": "dmcp", + "tools": [ + { + "annotations": null, + "description": "Given a string of text describing a dice roll...", + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "diceRollExpression": { + "type": "string" + } + }, + "required": ["diceRollExpression"], + "additionalProperties": false + }, + "name": "roll" + } + ] + }, + { + "id": "mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339", + "type": "mcp_approval_request", + "arguments": "{\"diceRollExpression\":\"2d4 + 1\"}", + "name": "roll", + "server_label": "dmcp" + }, + { + "type": "mcp_approval_response", + "approve": true, + "approval_request_id": "mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339" + } + ] + }, + "status": 200, + "response": { + "id": "resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0", + "object": "response", + "created_at": 1741294021, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "id": "mcp_68a6102d8948819c9b1490d36d5ffa4a0679e572a900e618", + "type": "mcp_call", + "approval_request_id": null, + "arguments": "{\"diceRollExpression\":\"2d4 + 1\"}", + "error": null, + "name": "roll", + "output": "4", + "server_label": "dmcp" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [ + { + "type": "mcp", + "server_label": "dmcp", + "server_description": "A Dungeons and Dragons MCP server to assist with dice rolling.", + "server_url": "https://dmcp-server.deno.dev/sse", + "require_approval": "never", + "allowed_tools": ["roll"] + } + ], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 291, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 314 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:40 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-records-web-search-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-web-search-calls.json new file mode 100644 index 0000000000..6ab56292ab --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-records-web-search-calls.json @@ -0,0 +1,86 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "tools": [{ "type": "web_search" }], + "input": "What was a positive news story from today?", + "include": ["web_search_call.action.sources"] + }, + "status": 200, + "response": { + "id": "resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0", + "object": "response", + "created_at": 1741294021, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "type": "web_search_call", + "id": "ws_67c9fa0502748190b7dd390736892e100be649c1a5ff9609", + "action": "search", + "status": "completed" + }, + { + "id": "msg_67c9fa077e288190af08fdffda2e34f20be649c1a5ff9609", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "On March 6, 2025, several news...", + "annotations": [ + { + "type": "url_citation", + "start_index": 2606, + "end_index": 2758, + "url": "https://...", + "title": "Title..." + } + ] + } + ] + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [{ "type": "web_search" }], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 291, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 314 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:40 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-adds-genai-conventions.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-adds-genai-conventions.json new file mode 100644 index 0000000000..e6646ed895 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-adds-genai-conventions.json @@ -0,0 +1,73 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "Answer in up to 3 words: Which ocean contains Bouvet Island?" + }, + "status": 200, + "response": { + "id": "resp_67ccd2bed1ec8190b14f964abc0542670bb6a6b452d3795b", + "object": "response", + "created_at": 1741476542, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "type": "message", + "id": "msg_67ccd2bf17f0819081ff3bb2cf6508e60bb6a6b452d3795b", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Atlantic Ocean.", + "annotations": [] + } + ] + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 25 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:37 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-code-interpreter-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-code-interpreter-calls.json new file mode 100644 index 0000000000..b7d5908239 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-code-interpreter-calls.json @@ -0,0 +1,93 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "tools": [ + { + "type": "code_interpreter", + "container": "cfile_682e0e8a43c88191a7978f477a09bdf5" + } + ], + "instructions": null, + "input": "Plot and analyse the histogram of the RGB channels for the uploaded image." + }, + "status": 200, + "response": { + "id": "resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0", + "object": "response", + "created_at": 1741294021, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "id": "msg_682d514e268c8191a89c38ea318446200f2610a7ec781a4f", + "content": [ + { + "annotations": [ + { + "file_id": "cfile_682d514b2e00819184b9b07e13557f82", + "index": null, + "type": "container_file_citation", + "container_id": "cntr_682d513bb0c48191b10bd4f8b0b3312200e64562acc2e0af", + "end_index": 0, + "filename": "cfile_682d514b2e00819184b9b07e13557f82.png", + "start_index": 0 + } + ], + "text": "Here is the histogram of the RGB channels for the uploaded image. Each curve represents the distribution of pixel intensities for the red, green, and blue channels. Peaks toward the high end of the intensity scale (right-hand side) suggest a lot of brightness and strong warm tones, matching the orange and light background in the image. If you want a different style of histogram (e.g., overall intensity, or quantized color groups), let me know!", + "type": "output_text", + "logprobs": [] + } + ], + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [ + { + "type": "code_interpreter", + "container": "cfile_682e0e8a43c88191a7978f477a09bdf5" + } + ], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 291, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 314 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:40 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-computer-use-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-computer-use-calls.json new file mode 100644 index 0000000000..2d37648120 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-computer-use-calls.json @@ -0,0 +1,109 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "computer-use-preview", + "tools": [ + { + "type": "computer_use_preview", + "display_width": 1024, + "display_height": 768, + "environment": "browser" + } + ], + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Check the latest OpenAI news on bing.com." + } + ] + } + ], + "reasoning": { + "summary": "concise" + }, + "truncation": "auto" + }, + "status": 200, + "response": { + "id": "resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0", + "object": "response", + "created_at": 1741294021, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "computer-use-preview-2025-04-14", + "output": [ + { + "type": "reasoning", + "id": "rs_67cc...", + "summary": [ + { + "type": "summary_text", + "text": "Clicking on the browser address bar." + } + ] + }, + { + "type": "computer_call", + "id": "cu_67cc...", + "call_id": "call_zw3...", + "action": { + "type": "click", + "button": "left", + "x": 156, + "y": 50 + }, + "pending_safety_checks": [], + "status": "completed" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [ + { + "type": "computer_use_preview", + "display_width": 1024, + "display_height": 768, + "environment": "browser" + } + ], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 291, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 314 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:40 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-custom-tool-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-custom-tool-calls.json new file mode 100644 index 0000000000..7d2e057f6e --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-custom-tool-calls.json @@ -0,0 +1,84 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "Use the code_exec tool to print hello world to the console.", + "tools": [ + { + "type": "custom", + "name": "code_exec", + "description": "Executes arbitrary Python code." + } + ] + }, + "status": 200, + "response": { + "id": "resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0", + "object": "response", + "created_at": 1741294021, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "id": "rs_6890e972fa7c819ca8bc561526b989170694874912ae0ea6", + "type": "reasoning", + "content": [], + "summary": [] + }, + { + "id": "ctc_6890e975e86c819c9338825b3e1994810694874912ae0ea6", + "type": "custom_tool_call", + "status": "completed", + "call_id": "call_aGiFQkRWSWAIsMQ19fKqxUgb", + "input": "print(\"hello world\")", + "name": "code_exec" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [ + { + "type": "custom", + "name": "code_exec", + "description": "Executes arbitrary Python code." + } + ], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 291, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 314 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:40 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-file-search-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-file-search-calls.json new file mode 100644 index 0000000000..8a372dd8e2 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-file-search-calls.json @@ -0,0 +1,109 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "What is deep research by OpenAI?", + "tools": [ + { + "type": "file_search", + "vector_store_ids": [""], + "max_num_results": 2 + } + ], + "include": ["file_search_call.results"] + }, + "status": 200, + "response": { + "id": "resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0", + "object": "response", + "created_at": 1741294021, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "type": "file_search_call", + "id": "fs_67c09ccea8c48191ade9367e3ba71515", + "status": "completed", + "queries": ["What is deep research?"], + "results": [ + { + "id": "file-2dtbBZdjtDKS8eqWxqbgDi", + "filename": "deep_research_blog.pdf", + "score": 0.95, + "text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed euismod, nisl eget aliquam aliquet, nunc nisl aliquet nisl, eget aliquam nisl nisl eget nisl. Sed euismod, nisl eget aliquam aliquet, nunc nisl aliquet nisl, eget aliquam nisl nisl eget nisl." + } + ] + }, + { + "id": "msg_67c09cd3091c819185af2be5d13d87de", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Deep research is a sophisticated capability that allows for extensive inquiry and synthesis of information across various domains. It is designed to conduct multi-step research tasks, gather data from multiple online sources, and provide comprehensive reports similar to what a research analyst would produce. This functionality is particularly useful in fields requiring detailed and accurate information...", + "annotations": [ + { + "type": "file_citation", + "index": 992, + "file_id": "file-2dtbBZdjtDKS8eqWxqbgDi", + "filename": "deep_research_blog.pdf" + }, + { + "type": "file_citation", + "index": 1176, + "file_id": "file-2dtbBZdjtDKS8eqWxqbgDi", + "filename": "deep_research_blog.pdf" + } + ] + } + ] + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [ + { + "type": "file_search", + "vector_store_ids": [""] + } + ], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 291, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 314 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:40 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-function-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-function-calls.json new file mode 100644 index 0000000000..aaf76241c9 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-function-calls.json @@ -0,0 +1,161 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": [ + { + "role": "system", + "content": "You are a helpful assistant providing weather updates." + }, + { + "role": "user", + "content": "What is the weather in Paris and Bogotá?" + } + ], + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city, e.g. San Francisco" + } + }, + "required": ["location"] + }, + "strict": true + }, + { + "type": "function", + "name": "send_email", + "description": "Send an email", + "parameters": { + "type": "object", + "properties": { + "to": { + "type": "string", + "description": "The email address of the recipient" + }, + "body": { + "type": "string", + "description": "The body of the email" + } + }, + "required": ["to", "body"] + }, + "strict": true + } + ], + "parallel_tool_calls": true + }, + "status": 200, + "response": { + "id": "resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0", + "object": "response", + "created_at": 1741294021, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "fc_12345xyz", + "call_id": "call_12345xyz", + "type": "function_call", + "name": "get_weather", + "arguments": "{\"location\":\"Paris, France\"}" + }, + { + "id": "fc_67890abc", + "call_id": "call_67890abc", + "type": "function_call", + "name": "get_weather", + "arguments": "{\"location\":\"Bogotá, Colombia\"}" + }, + { + "id": "fc_99999def", + "call_id": "call_99999def", + "type": "function_call", + "name": "send_email", + "arguments": "{\"to\":\"bob@email.com\",\"body\":\"Hi bob\"}" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and country, e.g. San Francisco, USA" + } + }, + "required": ["location"] + } + }, + { + "type": "function", + "name": "send_email", + "description": "Send an email", + "parameters": { + "type": "object", + "properties": { + "to": { + "type": "string", + "description": "The email address of the recipient" + }, + "body": { + "type": "string", + "description": "The body of the email" + } + }, + "required": ["to", "body"] + } + } + ], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 291, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 314 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:40 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-image-generation-call.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-image-generation-call.json new file mode 100644 index 0000000000..4363e06dc0 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-image-generation-call.json @@ -0,0 +1,76 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": [ + { + "role": "user", + "content": [ + { "type": "input_text", "text": "Now make it look realistic" } + ] + }, + { + "type": "image_generation_call", + "id": "ig_123" + } + ], + "tools": [{ "type": "image_generation" }] + }, + "status": 200, + "response": { + "id": "resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0", + "object": "response", + "created_at": 1741294021, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "id": "ig_124", + "type": "image_generation_call", + "status": "completed", + "revised_prompt": "A gray tabby cat hugging an otter. The otter is wearing an orange scarf. Both animals are cute and friendly, depicted in a warm, heartwarming, but realistic style.", + "result": "..." + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [{ "type": "image_generation" }], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 291, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 314 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:40 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-mcp-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-mcp-calls.json new file mode 100644 index 0000000000..759579484a --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-mcp-calls.json @@ -0,0 +1,126 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "tools": [ + { + "type": "mcp", + "server_label": "dmcp", + "server_description": "A Dungeons and Dragons MCP server to assist with dice rolling.", + "server_url": "https://dmcp-server.deno.dev/sse", + "require_approval": "never", + "allowed_tools": ["roll"] + } + ], + "input": [ + { + "role": "user", + "content": "Roll 2d4+1" + }, + { + "id": "mcpl_68a6102a4968819c8177b05584dd627b0679e572a900e618", + "type": "mcp_list_tools", + "server_label": "dmcp", + "tools": [ + { + "annotations": null, + "description": "Given a string of text describing a dice roll...", + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "diceRollExpression": { + "type": "string" + } + }, + "required": ["diceRollExpression"], + "additionalProperties": false + }, + "name": "roll" + } + ] + }, + { + "id": "mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339", + "type": "mcp_approval_request", + "arguments": "{\"diceRollExpression\":\"2d4 + 1\"}", + "name": "roll", + "server_label": "dmcp" + }, + { + "type": "mcp_approval_response", + "approve": true, + "approval_request_id": "mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339" + } + ] + }, + "status": 200, + "response": { + "id": "resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0", + "object": "response", + "created_at": 1741294021, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "id": "mcp_68a6102d8948819c9b1490d36d5ffa4a0679e572a900e618", + "type": "mcp_call", + "approval_request_id": null, + "arguments": "{\"diceRollExpression\":\"2d4 + 1\"}", + "error": null, + "name": "roll", + "output": "4", + "server_label": "dmcp" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [ + { + "type": "mcp", + "server_label": "dmcp", + "server_description": "A Dungeons and Dragons MCP server to assist with dice rolling.", + "server_url": "https://dmcp-server.deno.dev/sse", + "require_approval": "never", + "allowed_tools": ["roll"] + } + ], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 291, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 314 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:40 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-web-search-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-web-search-calls.json new file mode 100644 index 0000000000..6ab56292ab --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-responses-with-content-capture-records-web-search-calls.json @@ -0,0 +1,86 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "tools": [{ "type": "web_search" }], + "input": "What was a positive news story from today?", + "include": ["web_search_call.action.sources"] + }, + "status": 200, + "response": { + "id": "resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0", + "object": "response", + "created_at": 1741294021, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "type": "web_search_call", + "id": "ws_67c9fa0502748190b7dd390736892e100be649c1a5ff9609", + "action": "search", + "status": "completed" + }, + { + "id": "msg_67c9fa077e288190af08fdffda2e34f20be649c1a5ff9609", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "On March 6, 2025, several news...", + "annotations": [ + { + "type": "url_citation", + "start_index": 2606, + "end_index": 2758, + "url": "https://...", + "title": "Title..." + } + ] + } + ] + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + } + }, + "tool_choice": "auto", + "tools": [{ "type": "web_search" }], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 291, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 314 + }, + "user": null, + "metadata": {} + }, + "rawHeaders": { + "content-type": "application/json", + "date": "Fri, 11 Jul 2025 07:02:40 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-adds-genai-conventions.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-adds-genai-conventions.json new file mode 100644 index 0000000000..49002bce68 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-adds-genai-conventions.json @@ -0,0 +1,19 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "Answer in up to 3 words: Which ocean contains Bouvet Island?", + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_stream_adds\",\"object\":\"response\",\"created_at\":1741476542,\"model\":\"gpt-4o-mini-2024-07-18\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"in_progress\",\"content\":[{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}]}}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"Atlantic \",\"item_id\":\"msg_1\",\"sequence_number\":3}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"Ocean.\",\"item_id\":\"msg_1\",\"sequence_number\":4}\n\ndata: {\"type\":\"response.output_text.done\",\"output_index\":0,\"content_index\":0,\"text\":\"Atlantic Ocean.\",\"item_id\":\"msg_1\",\"sequence_number\":5}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"Atlantic Ocean.\",\"annotations\":[]}]},\"sequence_number\":6}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_stream_adds\",\"object\":\"response\",\"created_at\":1741476542,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"Atlantic Ocean.\",\"annotations\":[]}]}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":22,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":3,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":25},\"user\":null,\"metadata\":{}},\"sequence_number\":7}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-does-not-misbehave-with-double-iteration.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-does-not-misbehave-with-double-iteration.json new file mode 100644 index 0000000000..51b8cf5cfb --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-does-not-misbehave-with-double-iteration.json @@ -0,0 +1,19 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "Answer in up to 3 words: Which ocean contains Bouvet Island?", + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_stream_double\",\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"in_progress\",\"content\":[{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}]}}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"South \",\"item_id\":\"msg_1\",\"sequence_number\":3}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"Atlantic \",\"item_id\":\"msg_1\",\"sequence_number\":4}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"Ocean.\",\"item_id\":\"msg_1\",\"sequence_number\":5}\n\ndata: {\"type\":\"response.output_text.done\",\"output_index\":0,\"content_index\":0,\"text\":\"South Atlantic Ocean.\",\"item_id\":\"msg_1\",\"sequence_number\":6}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"South Atlantic Ocean.\",\"annotations\":[]}]},\"sequence_number\":7}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_stream_double\",\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"South Atlantic Ocean.\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":22,\"output_tokens\":4,\"total_tokens\":26}}}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 08:30:00 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-code-interpreter-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-code-interpreter-calls.json new file mode 100644 index 0000000000..da5266ac1d --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-code-interpreter-calls.json @@ -0,0 +1,26 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "tools": [ + { + "type": "code_interpreter", + "container": "cfile_682e0e8a43c88191a7978f477a09bdf5" + } + ], + "instructions": null, + "input": "Plot and analyse the histogram of the RGB channels for the uploaded image.", + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"model\":\"gpt-4.1-2025-04-14\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"msg_682d514e268c8191a89c38ea318446200f2610a7ec781a4f\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"Here is the histogram of the RGB channels for the uploaded image. Each curve represents the distribution of pixel intensities for the red, green, and blue channels. Peaks toward the high end of the intensity scale (right-hand side) suggest a lot of brightness and strong warm tones, matching the orange and light background in the image. If you want a different style of histogram (e.g., overall intensity, or quantized color groups), let me know!\",\"annotations\":[{\"file_id\":\"cfile_682d514b2e00819184b9b07e13557f82\",\"index\":null,\"type\":\"container_file_citation\",\"container_id\":\"cntr_682d513bb0c48191b10bd4f8b0b3312200e64562acc2e0af\",\"end_index\":0,\"filename\":\"cfile_682d514b2e00819184b9b07e13557f82.png\",\"start_index\":0}]}]},\"sequence_number\":1}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"gpt-4.1-2025-04-14\",\"output\":[{\"id\":\"msg_682d514e268c8191a89c38ea318446200f2610a7ec781a4f\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"Here is the histogram of the RGB channels for the uploaded image. Each curve represents the distribution of pixel intensities for the red, green, and blue channels. Peaks toward the high end of the intensity scale (right-hand side) suggest a lot of brightness and strong warm tones, matching the orange and light background in the image. If you want a different style of histogram (e.g., overall intensity, or quantized color groups), let me know!\",\"annotations\":[{\"file_id\":\"cfile_682d514b2e00819184b9b07e13557f82\",\"index\":null,\"type\":\"container_file_citation\",\"container_id\":\"cntr_682d513bb0c48191b10bd4f8b0b3312200e64562acc2e0af\",\"end_index\":0,\"filename\":\"cfile_682d514b2e00819184b9b07e13557f82.png\",\"start_index\":0}]}]}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"code_interpreter\",\"container\":\"cfile_682e0e8a43c88191a7978f477a09bdf5\"}],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":291,\"output_tokens\":23,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":314},\"user\":null,\"metadata\":{}},\"sequence_number\":2}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-computer-use-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-computer-use-calls.json new file mode 100644 index 0000000000..eb73bb767f --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-computer-use-calls.json @@ -0,0 +1,41 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "computer-use-preview", + "tools": [ + { + "type": "computer_use_preview", + "display_width": 1024, + "display_height": 768, + "environment": "browser" + } + ], + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Check the latest OpenAI news on bing.com." + } + ] + } + ], + "reasoning": { + "summary": "concise" + }, + "truncation": "auto", + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"model\":\"computer-use-preview-2025-04-14\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"rs_67cc...\",\"type\":\"reasoning\",\"status\":\"completed\",\"summary\":[{\"type\":\"summary_text\",\"text\":\"Clicking on the browser address bar.\"}]},\"sequence_number\":1}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":1,\"item\":{\"id\":\"cu_67cc...\",\"type\":\"computer_call\",\"status\":\"completed\",\"call_id\":\"call_zw3...\",\"action\":{\"type\":\"click\",\"button\":\"left\",\"x\":156,\"y\":50},\"pending_safety_checks\":[]},\"sequence_number\":2}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"computer-use-preview-2025-04-14\",\"output\":[{\"id\":\"rs_67cc...\",\"type\":\"reasoning\",\"summary\":[{\"type\":\"summary_text\",\"text\":\"Clicking on the browser address bar.\"}]},{\"id\":\"cu_67cc...\",\"type\":\"computer_call\",\"status\":\"completed\",\"call_id\":\"call_zw3...\",\"action\":{\"type\":\"click\",\"button\":\"left\",\"x\":156,\"y\":50},\"pending_safety_checks\":[]}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"computer_use_preview\",\"display_width\":1024,\"display_height\":768,\"environment\":\"browser\"}],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":291,\"output_tokens\":23,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":314},\"user\":null,\"metadata\":{}},\"sequence_number\":3}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-custom-tool-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-custom-tool-calls.json new file mode 100644 index 0000000000..108afcec81 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-custom-tool-calls.json @@ -0,0 +1,26 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "Use the code_exec tool to print hello world to the console.", + "tools": [ + { + "type": "custom", + "name": "code_exec", + "description": "Executes arbitrary Python code." + } + ], + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"model\":\"gpt-4.1-2025-04-14\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"rs_6890e972fa7c819ca8bc561526b989170694874912ae0ea6\",\"type\":\"reasoning\",\"status\":\"completed\",\"content\":[],\"summary\":[]},\"sequence_number\":1}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":1,\"item\":{\"id\":\"ctc_6890e975e86c819c9338825b3e1994810694874912ae0ea6\",\"type\":\"custom_tool_call\",\"status\":\"completed\",\"call_id\":\"call_aGiFQkRWSWAIsMQ19fKqxUgb\",\"input\":\"print(\\\"hello world\\\")\",\"name\":\"code_exec\"},\"sequence_number\":2}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"gpt-4.1-2025-04-14\",\"output\":[{\"id\":\"rs_6890e972fa7c819ca8bc561526b989170694874912ae0ea6\",\"type\":\"reasoning\",\"content\":[],\"summary\":[]},{\"id\":\"ctc_6890e975e86c819c9338825b3e1994810694874912ae0ea6\",\"type\":\"custom_tool_call\",\"status\":\"completed\",\"call_id\":\"call_aGiFQkRWSWAIsMQ19fKqxUgb\",\"input\":\"print(\\\"hello world\\\")\",\"name\":\"code_exec\"}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"custom\",\"name\":\"code_exec\",\"description\":\"Executes arbitrary Python code.\"}],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":291,\"output_tokens\":23,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":314},\"user\":null,\"metadata\":{}},\"sequence_number\":3}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-file-search-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-file-search-calls.json new file mode 100644 index 0000000000..044913e8f1 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-file-search-calls.json @@ -0,0 +1,27 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "What is deep research by OpenAI?", + "tools": [ + { + "type": "file_search", + "vector_store_ids": [""], + "max_num_results": 2 + } + ], + "include": ["file_search_call.results"], + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"model\":\"gpt-4.1-2025-04-14\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"fs_67c09ccea8c48191ade9367e3ba71515\",\"type\":\"file_search_call\",\"status\":\"completed\",\"queries\":[\"What is deep research?\"],\"results\":[{\"id\":\"file-2dtbBZdjtDKS8eqWxqbgDi\",\"filename\":\"deep_research_blog.pdf\",\"score\":0.95,\"text\":\"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed euismod, nisl eget aliquam aliquet, nunc nisl aliquet nisl, eget aliquam nisl nisl eget nisl. Sed euismod, nisl eget aliquam aliquet, nunc nisl aliquet nisl, eget aliquam nisl nisl eget nisl.\"}]},\"sequence_number\":1}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":1,\"item\":{\"id\":\"msg_67c09cd3091c819185af2be5d13d87de\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"Deep research is a sophisticated capability that allows for extensive inquiry and synthesis of information across various domains. It is designed to conduct multi-step research tasks, gather data from multiple online sources, and provide comprehensive reports similar to what a research analyst would produce. This functionality is particularly useful in fields requiring detailed and accurate information...\",\"annotations\":[{\"type\":\"file_citation\",\"index\":992,\"file_id\":\"file-2dtbBZdjtDKS8eqWxqbgDi\",\"filename\":\"deep_research_blog.pdf\"},{\"type\":\"file_citation\",\"index\":1176,\"file_id\":\"file-2dtbBZdjtDKS8eqWxqbgDi\",\"filename\":\"deep_research_blog.pdf\"}]}]},\"sequence_number\":2}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"gpt-4.1-2025-04-14\",\"output\":[{\"id\":\"fs_67c09ccea8c48191ade9367e3ba71515\",\"type\":\"file_search_call\",\"status\":\"completed\",\"queries\":[\"What is deep research?\"],\"results\":[{\"id\":\"file-2dtbBZdjtDKS8eqWxqbgDi\",\"filename\":\"deep_research_blog.pdf\",\"score\":0.95,\"text\":\"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed euismod, nisl eget aliquam aliquet, nunc nisl aliquet nisl, eget aliquam nisl nisl eget nisl. Sed euismod, nisl eget aliquam aliquet, nunc nisl aliquet nisl, eget aliquam nisl nisl eget nisl.\"}]},{\"id\":\"msg_67c09cd3091c819185af2be5d13d87de\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"Deep research is a sophisticated capability that allows for extensive inquiry and synthesis of information across various domains. It is designed to conduct multi-step research tasks, gather data from multiple online sources, and provide comprehensive reports similar to what a research analyst would produce. This functionality is particularly useful in fields requiring detailed and accurate information...\",\"annotations\":[{\"type\":\"file_citation\",\"index\":992,\"file_id\":\"file-2dtbBZdjtDKS8eqWxqbgDi\",\"filename\":\"deep_research_blog.pdf\"},{\"type\":\"file_citation\",\"index\":1176,\"file_id\":\"file-2dtbBZdjtDKS8eqWxqbgDi\",\"filename\":\"deep_research_blog.pdf\"}]}]}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"file_search\",\"vector_store_ids\":[\"\"]}],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":291,\"output_tokens\":23,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":314},\"user\":null,\"metadata\":{}},\"sequence_number\":3}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-function-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-function-calls.json new file mode 100644 index 0000000000..7da930f80f --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-function-calls.json @@ -0,0 +1,67 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": [ + { + "role": "system", + "content": "You are a helpful assistant providing weather updates." + }, + { + "role": "user", + "content": "What is the weather in Paris and Bogotá?" + } + ], + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city, e.g. San Francisco" + } + }, + "required": ["location"] + }, + "strict": true + }, + { + "type": "function", + "name": "send_email", + "description": "Send an email", + "parameters": { + "type": "object", + "properties": { + "to": { + "type": "string", + "description": "The email address of the recipient" + }, + "body": { + "type": "string", + "description": "The body of the email" + } + }, + "required": ["to", "body"] + }, + "strict": true + } + ], + "parallel_tool_calls": true, + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"model\":\"gpt-4o-mini-2024-07-18\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"fc_12345xyz\",\"call_id\":\"call_12345xyz\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Paris, France\\\"}\",\"status\":\"completed\"},\"sequence_number\":1}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":1,\"item\":{\"id\":\"fc_67890abc\",\"call_id\":\"call_67890abc\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Bogotá, Colombia\\\"}\",\"status\":\"completed\"},\"sequence_number\":2}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":2,\"item\":{\"id\":\"fc_99999def\",\"call_id\":\"call_99999def\",\"type\":\"function_call\",\"name\":\"send_email\",\"arguments\":\"{\\\"to\\\":\\\"bob@email.com\\\",\\\"body\\\":\\\"Hi bob\\\"}\",\"status\":\"completed\"},\"sequence_number\":3}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[{\"id\":\"fc_12345xyz\",\"call_id\":\"call_12345xyz\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"},{\"id\":\"fc_67890abc\",\"call_id\":\"call_67890abc\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Bogotá, Colombia\\\"}\"},{\"id\":\"fc_99999def\",\"call_id\":\"call_99999def\",\"type\":\"function_call\",\"name\":\"send_email\",\"arguments\":\"{\\\"to\\\":\\\"bob@email.com\\\",\\\"body\\\":\\\"Hi bob\\\"}\"}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the current weather in a given location\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country, e.g. San Francisco, USA\"}},\"required\":[\"location\"]}},{\"type\":\"function\",\"name\":\"send_email\",\"description\":\"Send an email\",\"parameters\":{\"type\":\"object\",\"properties\":{\"to\":{\"type\":\"string\",\"description\":\"The email address of the recipient\"},\"body\":{\"type\":\"string\",\"description\":\"The body of the email\"}},\"required\":[\"to\",\"body\"]}}],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":291,\"output_tokens\":23,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":314},\"user\":null,\"metadata\":{}},\"sequence_number\":4}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-image-generation-call.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-image-generation-call.json new file mode 100644 index 0000000000..07d31cfac5 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-image-generation-call.json @@ -0,0 +1,38 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Now make it look realistic" + } + ] + }, + { + "type": "image_generation_call", + "id": "ig_123" + } + ], + "tools": [ + { + "type": "image_generation" + } + ], + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"model\":\"gpt-4.1-2025-04-14\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"ig_124\",\"type\":\"image_generation_call\",\"status\":\"completed\",\"revised_prompt\":\"A gray tabby cat hugging an otter. The otter is wearing an orange scarf. Both animals are cute and friendly, depicted in a warm, heartwarming, but realistic style.\",\"result\":\"...\"},\"sequence_number\":1}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"gpt-4.1-2025-04-14\",\"output\":[{\"id\":\"ig_124\",\"type\":\"image_generation_call\",\"status\":\"completed\",\"revised_prompt\":\"A gray tabby cat hugging an otter. The otter is wearing an orange scarf. Both animals are cute and friendly, depicted in a warm, heartwarming, but realistic style.\",\"result\":\"...\"}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"image_generation\"}],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":291,\"output_tokens\":23,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":314},\"user\":null,\"metadata\":{}},\"sequence_number\":2}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-mcp-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-mcp-calls.json new file mode 100644 index 0000000000..3af9b6a082 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-mcp-calls.json @@ -0,0 +1,69 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "tools": [ + { + "type": "mcp", + "server_label": "dmcp", + "server_description": "A Dungeons and Dragons MCP server to assist with dice rolling.", + "server_url": "https://dmcp-server.deno.dev/sse", + "require_approval": "never", + "allowed_tools": ["roll"] + } + ], + "input": [ + { + "role": "user", + "content": "Roll 2d4+1" + }, + { + "id": "mcpl_68a6102a4968819c8177b05584dd627b0679e572a900e618", + "type": "mcp_list_tools", + "server_label": "dmcp", + "tools": [ + { + "annotations": null, + "description": "Given a string of text describing a dice roll...", + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "diceRollExpression": { + "type": "string" + } + }, + "required": ["diceRollExpression"], + "additionalProperties": false + }, + "name": "roll" + } + ] + }, + { + "id": "mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339", + "type": "mcp_approval_request", + "arguments": "{\"diceRollExpression\":\"2d4 + 1\"}", + "name": "roll", + "server_label": "dmcp" + }, + { + "type": "mcp_approval_response", + "approve": true, + "approval_request_id": "mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339" + } + ], + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"model\":\"gpt-4.1-2025-04-14\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"mcp_68a6102d8948819c9b1490d36d5ffa4a0679e572a900e618\",\"type\":\"mcp_call\",\"approval_request_id\":null,\"arguments\":\"{\\\"diceRollExpression\\\":\\\"2d4 + 1\\\"}\",\"error\":null,\"name\":\"roll\",\"output\":\"4\",\"server_label\":\"dmcp\"},\"sequence_number\":1}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"gpt-4.1-2025-04-14\",\"output\":[{\"id\":\"mcp_68a6102d8948819c9b1490d36d5ffa4a0679e572a900e618\",\"type\":\"mcp_call\",\"approval_request_id\":null,\"arguments\":\"{\\\"diceRollExpression\\\":\\\"2d4 + 1\\\"}\",\"error\":null,\"name\":\"roll\",\"output\":\"4\",\"server_label\":\"dmcp\"}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"mcp\",\"server_label\":\"dmcp\",\"server_description\":\"A Dungeons and Dragons MCP server to assist with dice rolling.\",\"server_url\":\"https://dmcp-server.deno.dev/sse\",\"require_approval\":\"never\",\"allowed_tools\":[\"roll\"]}],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":291,\"output_tokens\":23,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":314},\"user\":null,\"metadata\":{}},\"sequence_number\":2}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-tool-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-tool-calls.json new file mode 100644 index 0000000000..d408699236 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-tool-calls.json @@ -0,0 +1,39 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "Answer in up to 3 words: Which ocean contains Bouvet Island?", + "stream": true, + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": "Get the current weather in a city.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "strict": true + } + ], + "parallel_tool_calls": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_stream_tools\",\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"id\":\"tool_call_1\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"\",\"status\":\"in_progress\"}}\n\ndata: {\"type\":\"response.function_call_arguments.delta\",\"output_index\":0,\"item_id\":\"tool_call_1\",\"delta\":\"{\\\"location\\\":\\\"New York City\\\"}\",\"sequence_number\":2}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"tool_call_1\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"New York City\\\"}\",\"status\":\"completed\"},\"sequence_number\":3}\n\ndata: {\"type\":\"response.output_item.added\",\"output_index\":1,\"item\":{\"id\":\"tool_call_2\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"\",\"status\":\"in_progress\"}}\n\ndata: {\"type\":\"response.function_call_arguments.delta\",\"output_index\":1,\"item_id\":\"tool_call_2\",\"delta\":\"{\\\"location\\\":\\\"London\\\"}\",\"sequence_number\":4}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":1,\"item\":{\"id\":\"tool_call_2\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"London\\\"}\",\"status\":\"completed\"},\"sequence_number\":5}\n\ndata: {\"type\":\"response.output_item.added\",\"output_index\":2,\"item\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"in_progress\",\"content\":[{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}]}}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":2,\"content_index\":0,\"delta\":\"New York City is 25 degrees and sunny, while London is 15 degrees and raining.\",\"item_id\":\"msg_1\",\"sequence_number\":6}\n\ndata: {\"type\":\"response.output_text.done\",\"output_index\":2,\"content_index\":0,\"text\":\"New York City is 25 degrees and sunny, while London is 15 degrees and raining.\",\"item_id\":\"msg_1\",\"sequence_number\":7}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":2,\"item\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"New York City is 25 degrees and sunny, while London is 15 degrees and raining.\",\"annotations\":[]}]},\"sequence_number\":8}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_stream_tools\",\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[{\"id\":\"tool_call_1\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"New York City\\\"}\",\"status\":\"completed\"},{\"id\":\"tool_call_2\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"London\\\"}\",\"status\":\"completed\"},{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"New York City is 25 degrees and sunny, while London is 15 degrees and raining.\",\"annotations\":[]}]}]}}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 08:20:00 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-usage.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-usage.json new file mode 100644 index 0000000000..a1688b748c --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-usage.json @@ -0,0 +1,19 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "Answer in up to 3 words: Which ocean contains Bouvet Island?", + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_stream_usage\",\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"in_progress\",\"content\":[{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}]}}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"South \",\"item_id\":\"msg_1\",\"sequence_number\":3}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"Atlantic \",\"item_id\":\"msg_1\",\"sequence_number\":4}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"Ocean.\",\"item_id\":\"msg_1\",\"sequence_number\":5}\n\ndata: {\"type\":\"response.output_text.done\",\"output_index\":0,\"content_index\":0,\"text\":\"South Atlantic Ocean.\",\"item_id\":\"msg_1\",\"sequence_number\":6}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"South Atlantic Ocean.\",\"annotations\":[]}]},\"sequence_number\":7}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_stream_usage\",\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"South Atlantic Ocean.\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":22,\"output_tokens\":4,\"total_tokens\":26}}}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 08:02:29 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-web-search-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-web-search-calls.json new file mode 100644 index 0000000000..cf8762f107 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-records-web-search-calls.json @@ -0,0 +1,25 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "tools": [ + { + "type": "web_search" + } + ], + "input": "What was a positive news story from today?", + "include": ["web_search_call.action.sources"], + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"model\":\"gpt-4.1-2025-04-14\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"ws_67c9fa0502748190b7dd390736892e100be649c1a5ff9609\",\"type\":\"web_search_call\",\"action\":\"search\",\"status\":\"completed\"},\"sequence_number\":1}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":1,\"item\":{\"id\":\"msg_67c9fa077e288190af08fdffda2e34f20be649c1a5ff9609\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"On March 6, 2025, several news...\",\"annotations\":[{\"type\":\"url_citation\",\"start_index\":2606,\"end_index\":2758,\"url\":\"https://...\",\"title\":\"Title...\"}]}]},\"sequence_number\":2}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"gpt-4.1-2025-04-14\",\"output\":[{\"id\":\"ws_67c9fa0502748190b7dd390736892e100be649c1a5ff9609\",\"type\":\"web_search_call\",\"action\":\"search\",\"status\":\"completed\"},{\"id\":\"msg_67c9fa077e288190af08fdffda2e34f20be649c1a5ff9609\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"On March 6, 2025, several news...\",\"annotations\":[{\"type\":\"url_citation\",\"start_index\":2606,\"end_index\":2758,\"url\":\"https://...\",\"title\":\"Title...\"}]}]}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"web_search\"}],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":291,\"output_tokens\":23,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":314},\"user\":null,\"metadata\":{}},\"sequence_number\":3}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-adds-genai-conventions.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-adds-genai-conventions.json new file mode 100644 index 0000000000..49002bce68 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-adds-genai-conventions.json @@ -0,0 +1,19 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "Answer in up to 3 words: Which ocean contains Bouvet Island?", + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_stream_adds\",\"object\":\"response\",\"created_at\":1741476542,\"model\":\"gpt-4o-mini-2024-07-18\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"in_progress\",\"content\":[{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}]}}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"Atlantic \",\"item_id\":\"msg_1\",\"sequence_number\":3}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"Ocean.\",\"item_id\":\"msg_1\",\"sequence_number\":4}\n\ndata: {\"type\":\"response.output_text.done\",\"output_index\":0,\"content_index\":0,\"text\":\"Atlantic Ocean.\",\"item_id\":\"msg_1\",\"sequence_number\":5}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"Atlantic Ocean.\",\"annotations\":[]}]},\"sequence_number\":6}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_stream_adds\",\"object\":\"response\",\"created_at\":1741476542,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"Atlantic Ocean.\",\"annotations\":[]}]}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":22,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":3,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":25},\"user\":null,\"metadata\":{}},\"sequence_number\":7}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-does-not-misbehave-with-double-iteration.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-does-not-misbehave-with-double-iteration.json new file mode 100644 index 0000000000..51b8cf5cfb --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-does-not-misbehave-with-double-iteration.json @@ -0,0 +1,19 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "Answer in up to 3 words: Which ocean contains Bouvet Island?", + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_stream_double\",\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"in_progress\",\"content\":[{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}]}}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"South \",\"item_id\":\"msg_1\",\"sequence_number\":3}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"Atlantic \",\"item_id\":\"msg_1\",\"sequence_number\":4}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"Ocean.\",\"item_id\":\"msg_1\",\"sequence_number\":5}\n\ndata: {\"type\":\"response.output_text.done\",\"output_index\":0,\"content_index\":0,\"text\":\"South Atlantic Ocean.\",\"item_id\":\"msg_1\",\"sequence_number\":6}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"South Atlantic Ocean.\",\"annotations\":[]}]},\"sequence_number\":7}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_stream_double\",\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"South Atlantic Ocean.\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":22,\"output_tokens\":4,\"total_tokens\":26}}}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 08:30:00 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-code-interpreter-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-code-interpreter-calls.json new file mode 100644 index 0000000000..da5266ac1d --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-code-interpreter-calls.json @@ -0,0 +1,26 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "tools": [ + { + "type": "code_interpreter", + "container": "cfile_682e0e8a43c88191a7978f477a09bdf5" + } + ], + "instructions": null, + "input": "Plot and analyse the histogram of the RGB channels for the uploaded image.", + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"model\":\"gpt-4.1-2025-04-14\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"msg_682d514e268c8191a89c38ea318446200f2610a7ec781a4f\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"Here is the histogram of the RGB channels for the uploaded image. Each curve represents the distribution of pixel intensities for the red, green, and blue channels. Peaks toward the high end of the intensity scale (right-hand side) suggest a lot of brightness and strong warm tones, matching the orange and light background in the image. If you want a different style of histogram (e.g., overall intensity, or quantized color groups), let me know!\",\"annotations\":[{\"file_id\":\"cfile_682d514b2e00819184b9b07e13557f82\",\"index\":null,\"type\":\"container_file_citation\",\"container_id\":\"cntr_682d513bb0c48191b10bd4f8b0b3312200e64562acc2e0af\",\"end_index\":0,\"filename\":\"cfile_682d514b2e00819184b9b07e13557f82.png\",\"start_index\":0}]}]},\"sequence_number\":1}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"gpt-4.1-2025-04-14\",\"output\":[{\"id\":\"msg_682d514e268c8191a89c38ea318446200f2610a7ec781a4f\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"Here is the histogram of the RGB channels for the uploaded image. Each curve represents the distribution of pixel intensities for the red, green, and blue channels. Peaks toward the high end of the intensity scale (right-hand side) suggest a lot of brightness and strong warm tones, matching the orange and light background in the image. If you want a different style of histogram (e.g., overall intensity, or quantized color groups), let me know!\",\"annotations\":[{\"file_id\":\"cfile_682d514b2e00819184b9b07e13557f82\",\"index\":null,\"type\":\"container_file_citation\",\"container_id\":\"cntr_682d513bb0c48191b10bd4f8b0b3312200e64562acc2e0af\",\"end_index\":0,\"filename\":\"cfile_682d514b2e00819184b9b07e13557f82.png\",\"start_index\":0}]}]}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"code_interpreter\",\"container\":\"cfile_682e0e8a43c88191a7978f477a09bdf5\"}],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":291,\"output_tokens\":23,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":314},\"user\":null,\"metadata\":{}},\"sequence_number\":2}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-computer-use-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-computer-use-calls.json new file mode 100644 index 0000000000..eb73bb767f --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-computer-use-calls.json @@ -0,0 +1,41 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "computer-use-preview", + "tools": [ + { + "type": "computer_use_preview", + "display_width": 1024, + "display_height": 768, + "environment": "browser" + } + ], + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Check the latest OpenAI news on bing.com." + } + ] + } + ], + "reasoning": { + "summary": "concise" + }, + "truncation": "auto", + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"model\":\"computer-use-preview-2025-04-14\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"rs_67cc...\",\"type\":\"reasoning\",\"status\":\"completed\",\"summary\":[{\"type\":\"summary_text\",\"text\":\"Clicking on the browser address bar.\"}]},\"sequence_number\":1}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":1,\"item\":{\"id\":\"cu_67cc...\",\"type\":\"computer_call\",\"status\":\"completed\",\"call_id\":\"call_zw3...\",\"action\":{\"type\":\"click\",\"button\":\"left\",\"x\":156,\"y\":50},\"pending_safety_checks\":[]},\"sequence_number\":2}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"computer-use-preview-2025-04-14\",\"output\":[{\"id\":\"rs_67cc...\",\"type\":\"reasoning\",\"summary\":[{\"type\":\"summary_text\",\"text\":\"Clicking on the browser address bar.\"}]},{\"id\":\"cu_67cc...\",\"type\":\"computer_call\",\"status\":\"completed\",\"call_id\":\"call_zw3...\",\"action\":{\"type\":\"click\",\"button\":\"left\",\"x\":156,\"y\":50},\"pending_safety_checks\":[]}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"computer_use_preview\",\"display_width\":1024,\"display_height\":768,\"environment\":\"browser\"}],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":291,\"output_tokens\":23,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":314},\"user\":null,\"metadata\":{}},\"sequence_number\":3}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-custom-tool-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-custom-tool-calls.json new file mode 100644 index 0000000000..108afcec81 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-custom-tool-calls.json @@ -0,0 +1,26 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "Use the code_exec tool to print hello world to the console.", + "tools": [ + { + "type": "custom", + "name": "code_exec", + "description": "Executes arbitrary Python code." + } + ], + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"model\":\"gpt-4.1-2025-04-14\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"rs_6890e972fa7c819ca8bc561526b989170694874912ae0ea6\",\"type\":\"reasoning\",\"status\":\"completed\",\"content\":[],\"summary\":[]},\"sequence_number\":1}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":1,\"item\":{\"id\":\"ctc_6890e975e86c819c9338825b3e1994810694874912ae0ea6\",\"type\":\"custom_tool_call\",\"status\":\"completed\",\"call_id\":\"call_aGiFQkRWSWAIsMQ19fKqxUgb\",\"input\":\"print(\\\"hello world\\\")\",\"name\":\"code_exec\"},\"sequence_number\":2}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"gpt-4.1-2025-04-14\",\"output\":[{\"id\":\"rs_6890e972fa7c819ca8bc561526b989170694874912ae0ea6\",\"type\":\"reasoning\",\"content\":[],\"summary\":[]},{\"id\":\"ctc_6890e975e86c819c9338825b3e1994810694874912ae0ea6\",\"type\":\"custom_tool_call\",\"status\":\"completed\",\"call_id\":\"call_aGiFQkRWSWAIsMQ19fKqxUgb\",\"input\":\"print(\\\"hello world\\\")\",\"name\":\"code_exec\"}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"custom\",\"name\":\"code_exec\",\"description\":\"Executes arbitrary Python code.\"}],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":291,\"output_tokens\":23,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":314},\"user\":null,\"metadata\":{}},\"sequence_number\":3}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-file-search-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-file-search-calls.json new file mode 100644 index 0000000000..044913e8f1 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-file-search-calls.json @@ -0,0 +1,27 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "What is deep research by OpenAI?", + "tools": [ + { + "type": "file_search", + "vector_store_ids": [""], + "max_num_results": 2 + } + ], + "include": ["file_search_call.results"], + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"model\":\"gpt-4.1-2025-04-14\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"fs_67c09ccea8c48191ade9367e3ba71515\",\"type\":\"file_search_call\",\"status\":\"completed\",\"queries\":[\"What is deep research?\"],\"results\":[{\"id\":\"file-2dtbBZdjtDKS8eqWxqbgDi\",\"filename\":\"deep_research_blog.pdf\",\"score\":0.95,\"text\":\"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed euismod, nisl eget aliquam aliquet, nunc nisl aliquet nisl, eget aliquam nisl nisl eget nisl. Sed euismod, nisl eget aliquam aliquet, nunc nisl aliquet nisl, eget aliquam nisl nisl eget nisl.\"}]},\"sequence_number\":1}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":1,\"item\":{\"id\":\"msg_67c09cd3091c819185af2be5d13d87de\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"Deep research is a sophisticated capability that allows for extensive inquiry and synthesis of information across various domains. It is designed to conduct multi-step research tasks, gather data from multiple online sources, and provide comprehensive reports similar to what a research analyst would produce. This functionality is particularly useful in fields requiring detailed and accurate information...\",\"annotations\":[{\"type\":\"file_citation\",\"index\":992,\"file_id\":\"file-2dtbBZdjtDKS8eqWxqbgDi\",\"filename\":\"deep_research_blog.pdf\"},{\"type\":\"file_citation\",\"index\":1176,\"file_id\":\"file-2dtbBZdjtDKS8eqWxqbgDi\",\"filename\":\"deep_research_blog.pdf\"}]}]},\"sequence_number\":2}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"gpt-4.1-2025-04-14\",\"output\":[{\"id\":\"fs_67c09ccea8c48191ade9367e3ba71515\",\"type\":\"file_search_call\",\"status\":\"completed\",\"queries\":[\"What is deep research?\"],\"results\":[{\"id\":\"file-2dtbBZdjtDKS8eqWxqbgDi\",\"filename\":\"deep_research_blog.pdf\",\"score\":0.95,\"text\":\"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed euismod, nisl eget aliquam aliquet, nunc nisl aliquet nisl, eget aliquam nisl nisl eget nisl. Sed euismod, nisl eget aliquam aliquet, nunc nisl aliquet nisl, eget aliquam nisl nisl eget nisl.\"}]},{\"id\":\"msg_67c09cd3091c819185af2be5d13d87de\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"Deep research is a sophisticated capability that allows for extensive inquiry and synthesis of information across various domains. It is designed to conduct multi-step research tasks, gather data from multiple online sources, and provide comprehensive reports similar to what a research analyst would produce. This functionality is particularly useful in fields requiring detailed and accurate information...\",\"annotations\":[{\"type\":\"file_citation\",\"index\":992,\"file_id\":\"file-2dtbBZdjtDKS8eqWxqbgDi\",\"filename\":\"deep_research_blog.pdf\"},{\"type\":\"file_citation\",\"index\":1176,\"file_id\":\"file-2dtbBZdjtDKS8eqWxqbgDi\",\"filename\":\"deep_research_blog.pdf\"}]}]}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"file_search\",\"vector_store_ids\":[\"\"]}],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":291,\"output_tokens\":23,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":314},\"user\":null,\"metadata\":{}},\"sequence_number\":3}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-function-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-function-calls.json new file mode 100644 index 0000000000..7da930f80f --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-function-calls.json @@ -0,0 +1,67 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": [ + { + "role": "system", + "content": "You are a helpful assistant providing weather updates." + }, + { + "role": "user", + "content": "What is the weather in Paris and Bogotá?" + } + ], + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city, e.g. San Francisco" + } + }, + "required": ["location"] + }, + "strict": true + }, + { + "type": "function", + "name": "send_email", + "description": "Send an email", + "parameters": { + "type": "object", + "properties": { + "to": { + "type": "string", + "description": "The email address of the recipient" + }, + "body": { + "type": "string", + "description": "The body of the email" + } + }, + "required": ["to", "body"] + }, + "strict": true + } + ], + "parallel_tool_calls": true, + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"model\":\"gpt-4o-mini-2024-07-18\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"fc_12345xyz\",\"call_id\":\"call_12345xyz\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Paris, France\\\"}\",\"status\":\"completed\"},\"sequence_number\":1}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":1,\"item\":{\"id\":\"fc_67890abc\",\"call_id\":\"call_67890abc\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Bogotá, Colombia\\\"}\",\"status\":\"completed\"},\"sequence_number\":2}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":2,\"item\":{\"id\":\"fc_99999def\",\"call_id\":\"call_99999def\",\"type\":\"function_call\",\"name\":\"send_email\",\"arguments\":\"{\\\"to\\\":\\\"bob@email.com\\\",\\\"body\\\":\\\"Hi bob\\\"}\",\"status\":\"completed\"},\"sequence_number\":3}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[{\"id\":\"fc_12345xyz\",\"call_id\":\"call_12345xyz\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"},{\"id\":\"fc_67890abc\",\"call_id\":\"call_67890abc\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Bogotá, Colombia\\\"}\"},{\"id\":\"fc_99999def\",\"call_id\":\"call_99999def\",\"type\":\"function_call\",\"name\":\"send_email\",\"arguments\":\"{\\\"to\\\":\\\"bob@email.com\\\",\\\"body\\\":\\\"Hi bob\\\"}\"}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the current weather in a given location\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country, e.g. San Francisco, USA\"}},\"required\":[\"location\"]}},{\"type\":\"function\",\"name\":\"send_email\",\"description\":\"Send an email\",\"parameters\":{\"type\":\"object\",\"properties\":{\"to\":{\"type\":\"string\",\"description\":\"The email address of the recipient\"},\"body\":{\"type\":\"string\",\"description\":\"The body of the email\"}},\"required\":[\"to\",\"body\"]}}],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":291,\"output_tokens\":23,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":314},\"user\":null,\"metadata\":{}},\"sequence_number\":4}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-image-generation-call.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-image-generation-call.json new file mode 100644 index 0000000000..07d31cfac5 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-image-generation-call.json @@ -0,0 +1,38 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Now make it look realistic" + } + ] + }, + { + "type": "image_generation_call", + "id": "ig_123" + } + ], + "tools": [ + { + "type": "image_generation" + } + ], + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"model\":\"gpt-4.1-2025-04-14\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"ig_124\",\"type\":\"image_generation_call\",\"status\":\"completed\",\"revised_prompt\":\"A gray tabby cat hugging an otter. The otter is wearing an orange scarf. Both animals are cute and friendly, depicted in a warm, heartwarming, but realistic style.\",\"result\":\"...\"},\"sequence_number\":1}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"gpt-4.1-2025-04-14\",\"output\":[{\"id\":\"ig_124\",\"type\":\"image_generation_call\",\"status\":\"completed\",\"revised_prompt\":\"A gray tabby cat hugging an otter. The otter is wearing an orange scarf. Both animals are cute and friendly, depicted in a warm, heartwarming, but realistic style.\",\"result\":\"...\"}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"image_generation\"}],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":291,\"output_tokens\":23,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":314},\"user\":null,\"metadata\":{}},\"sequence_number\":2}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-mcp-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-mcp-calls.json new file mode 100644 index 0000000000..3af9b6a082 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-mcp-calls.json @@ -0,0 +1,69 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "tools": [ + { + "type": "mcp", + "server_label": "dmcp", + "server_description": "A Dungeons and Dragons MCP server to assist with dice rolling.", + "server_url": "https://dmcp-server.deno.dev/sse", + "require_approval": "never", + "allowed_tools": ["roll"] + } + ], + "input": [ + { + "role": "user", + "content": "Roll 2d4+1" + }, + { + "id": "mcpl_68a6102a4968819c8177b05584dd627b0679e572a900e618", + "type": "mcp_list_tools", + "server_label": "dmcp", + "tools": [ + { + "annotations": null, + "description": "Given a string of text describing a dice roll...", + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "diceRollExpression": { + "type": "string" + } + }, + "required": ["diceRollExpression"], + "additionalProperties": false + }, + "name": "roll" + } + ] + }, + { + "id": "mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339", + "type": "mcp_approval_request", + "arguments": "{\"diceRollExpression\":\"2d4 + 1\"}", + "name": "roll", + "server_label": "dmcp" + }, + { + "type": "mcp_approval_response", + "approve": true, + "approval_request_id": "mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339" + } + ], + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"model\":\"gpt-4.1-2025-04-14\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"mcp_68a6102d8948819c9b1490d36d5ffa4a0679e572a900e618\",\"type\":\"mcp_call\",\"approval_request_id\":null,\"arguments\":\"{\\\"diceRollExpression\\\":\\\"2d4 + 1\\\"}\",\"error\":null,\"name\":\"roll\",\"output\":\"4\",\"server_label\":\"dmcp\"},\"sequence_number\":1}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"gpt-4.1-2025-04-14\",\"output\":[{\"id\":\"mcp_68a6102d8948819c9b1490d36d5ffa4a0679e572a900e618\",\"type\":\"mcp_call\",\"approval_request_id\":null,\"arguments\":\"{\\\"diceRollExpression\\\":\\\"2d4 + 1\\\"}\",\"error\":null,\"name\":\"roll\",\"output\":\"4\",\"server_label\":\"dmcp\"}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"mcp\",\"server_label\":\"dmcp\",\"server_description\":\"A Dungeons and Dragons MCP server to assist with dice rolling.\",\"server_url\":\"https://dmcp-server.deno.dev/sse\",\"require_approval\":\"never\",\"allowed_tools\":[\"roll\"]}],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":291,\"output_tokens\":23,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":314},\"user\":null,\"metadata\":{}},\"sequence_number\":2}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-tool-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-tool-calls.json new file mode 100644 index 0000000000..d408699236 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-tool-calls.json @@ -0,0 +1,39 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "Answer in up to 3 words: Which ocean contains Bouvet Island?", + "stream": true, + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": "Get the current weather in a city.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "strict": true + } + ], + "parallel_tool_calls": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_stream_tools\",\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"id\":\"tool_call_1\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"\",\"status\":\"in_progress\"}}\n\ndata: {\"type\":\"response.function_call_arguments.delta\",\"output_index\":0,\"item_id\":\"tool_call_1\",\"delta\":\"{\\\"location\\\":\\\"New York City\\\"}\",\"sequence_number\":2}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"tool_call_1\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"New York City\\\"}\",\"status\":\"completed\"},\"sequence_number\":3}\n\ndata: {\"type\":\"response.output_item.added\",\"output_index\":1,\"item\":{\"id\":\"tool_call_2\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"\",\"status\":\"in_progress\"}}\n\ndata: {\"type\":\"response.function_call_arguments.delta\",\"output_index\":1,\"item_id\":\"tool_call_2\",\"delta\":\"{\\\"location\\\":\\\"London\\\"}\",\"sequence_number\":4}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":1,\"item\":{\"id\":\"tool_call_2\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"London\\\"}\",\"status\":\"completed\"},\"sequence_number\":5}\n\ndata: {\"type\":\"response.output_item.added\",\"output_index\":2,\"item\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"in_progress\",\"content\":[{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}]}}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":2,\"content_index\":0,\"delta\":\"New York City is 25 degrees and sunny, while London is 15 degrees and raining.\",\"item_id\":\"msg_1\",\"sequence_number\":6}\n\ndata: {\"type\":\"response.output_text.done\",\"output_index\":2,\"content_index\":0,\"text\":\"New York City is 25 degrees and sunny, while London is 15 degrees and raining.\",\"item_id\":\"msg_1\",\"sequence_number\":7}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":2,\"item\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"New York City is 25 degrees and sunny, while London is 15 degrees and raining.\",\"annotations\":[]}]},\"sequence_number\":8}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_stream_tools\",\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[{\"id\":\"tool_call_1\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"New York City\\\"}\",\"status\":\"completed\"},{\"id\":\"tool_call_2\",\"type\":\"function_call\",\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"London\\\"}\",\"status\":\"completed\"},{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"New York City is 25 degrees and sunny, while London is 15 degrees and raining.\",\"annotations\":[]}]}]}}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 08:20:00 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-usage.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-usage.json new file mode 100644 index 0000000000..a1688b748c --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-usage.json @@ -0,0 +1,19 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "input": "Answer in up to 3 words: Which ocean contains Bouvet Island?", + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_stream_usage\",\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"in_progress\",\"content\":[{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}]}}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"South \",\"item_id\":\"msg_1\",\"sequence_number\":3}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"Atlantic \",\"item_id\":\"msg_1\",\"sequence_number\":4}\n\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"Ocean.\",\"item_id\":\"msg_1\",\"sequence_number\":5}\n\ndata: {\"type\":\"response.output_text.done\",\"output_index\":0,\"content_index\":0,\"text\":\"South Atlantic Ocean.\",\"item_id\":\"msg_1\",\"sequence_number\":6}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"South Atlantic Ocean.\",\"annotations\":[]}]},\"sequence_number\":7}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_stream_usage\",\"model\":\"gpt-4o-mini-2024-07-18\",\"output\":[{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"South Atlantic Ocean.\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":22,\"output_tokens\":4,\"total_tokens\":26}}}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 08:02:29 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-web-search-calls.json b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-web-search-calls.json new file mode 100644 index 0000000000..cf8762f107 --- /dev/null +++ b/packages/instrumentation-openai/test/mock-responses/openai-streaming-responses-with-content-capture-records-web-search-calls.json @@ -0,0 +1,25 @@ +[ + { + "scope": "https://api.openai.com:443", + "method": "POST", + "path": "/v1/responses", + "body": { + "model": "gpt-4o-mini", + "tools": [ + { + "type": "web_search" + } + ], + "input": "What was a positive news story from today?", + "include": ["web_search_call.action.sources"], + "stream": true + }, + "status": 200, + "response": "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"model\":\"gpt-4.1-2025-04-14\",\"status\":\"in_progress\",\"output\":[]}}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"ws_67c9fa0502748190b7dd390736892e100be649c1a5ff9609\",\"type\":\"web_search_call\",\"action\":\"search\",\"status\":\"completed\"},\"sequence_number\":1}\n\ndata: {\"type\":\"response.output_item.done\",\"output_index\":1,\"item\":{\"id\":\"msg_67c9fa077e288190af08fdffda2e34f20be649c1a5ff9609\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"On March 6, 2025, several news...\",\"annotations\":[{\"type\":\"url_citation\",\"start_index\":2606,\"end_index\":2758,\"url\":\"https://...\",\"title\":\"Title...\"}]}]},\"sequence_number\":2}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_67ca09c5efe0819096d0511c92b8c890096610f474011cc0\",\"object\":\"response\",\"created_at\":1741294021,\"status\":\"completed\",\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"model\":\"gpt-4.1-2025-04-14\",\"output\":[{\"id\":\"ws_67c9fa0502748190b7dd390736892e100be649c1a5ff9609\",\"type\":\"web_search_call\",\"action\":\"search\",\"status\":\"completed\"},{\"id\":\"msg_67c9fa077e288190af08fdffda2e34f20be649c1a5ff9609\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"On March 6, 2025, several news...\",\"annotations\":[{\"type\":\"url_citation\",\"start_index\":2606,\"end_index\":2758,\"url\":\"https://...\",\"title\":\"Title...\"}]}]}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"web_search\"}],\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":291,\"output_tokens\":23,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":314},\"user\":null,\"metadata\":{}},\"sequence_number\":3}\n\ndata: [DONE]\n\n", + "rawHeaders": { + "content-type": "text/event-stream; charset=utf-8", + "date": "Thu, 17 Jul 2025 07:27:49 GMT" + }, + "responseIsBinary": false + } +] diff --git a/packages/instrumentation-openai/test/openai.test.ts b/packages/instrumentation-openai/test/openai.test.ts index 21402f2ce3..52ad05b492 100644 --- a/packages/instrumentation-openai/test/openai.test.ts +++ b/packages/instrumentation-openai/test/openai.test.ts @@ -45,9 +45,9 @@ import { ATTR_SERVER_PORT, } from '@opentelemetry/semantic-conventions'; import { expect } from 'expect'; -import { Definition, back as nockBack } from 'nock'; +import { type Definition, back as nockBack } from 'nock'; import { OpenAI } from 'openai'; -import * as path from 'path'; +import * as path from 'node:path'; import { ATTR_EVENT_NAME, @@ -56,6 +56,7 @@ import { ATTR_GEN_AI_REQUEST_ENCODING_FORMATS, ATTR_GEN_AI_REQUEST_FREQUENCY_PENALTY, ATTR_GEN_AI_REQUEST_MODEL, + ATTR_GEN_AI_PROVIDER_NAME, ATTR_GEN_AI_REQUEST_MAX_TOKENS, ATTR_GEN_AI_REQUEST_PRESENCE_PENALTY, ATTR_GEN_AI_REQUEST_TOP_P, @@ -66,7 +67,14 @@ import { ATTR_GEN_AI_REQUEST_STOP_SEQUENCES, ATTR_GEN_AI_REQUEST_TEMPERATURE, ATTR_GEN_AI_RESPONSE_MODEL, + ATTR_GEN_AI_INPUT_MESSAGES, + ATTR_GEN_AI_OUTPUT_MESSAGES, ATTR_GEN_AI_TOKEN_TYPE, + EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS, + GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + GEN_AI_OPERATION_NAME_VALUE_CHAT, + GEN_AI_TOKEN_TYPE_VALUE_INPUT, + GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, } from '../src/semconv'; // Remove any data from recorded responses that could have sensitive data @@ -3195,4 +3203,7171 @@ describe('OpenAI', function () { ).toBeGreaterThan(0); }); }); + + describe('responses', function () { + this.beforeEach(() => { + instrumentation.enable(); + }); + this.afterEach(() => { + instrumentation.disable(); + }); + + it('adds genai conventions', async () => { + const response = await client.responses.create({ + model, + input, + }); + + expect(response.output_text).toEqual('Atlantic Ocean.'); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 22, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 3, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 22, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 3, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + } + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [{ type: 'text', content: undefined }], + finish_reason: 'stop' + } + ]); + }); + + it('records all the client options', async () => { + const response = await client.responses.create({ + model, + input, + max_output_tokens: 100, + temperature: 1.0, + top_p: 1.0, + }); + + expect(response.output_text).toEqual('Southern Ocean.'); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_REQUEST_MAX_TOKENS]: 100, + [ATTR_GEN_AI_REQUEST_TEMPERATURE]: 1.0, + [ATTR_GEN_AI_REQUEST_TOP_P]: 1.0, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 22, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 3, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 22, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 3, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + } + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [{ type: 'text', content: undefined }], + finish_reason: 'stop' + } + ]); + }); + + it('records function calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'function', + name: 'get_weather', + description: 'Get the current weather in a given location', + parameters: { + type: 'object', + properties: { + location: { + type: 'string', + description: 'The city, e.g. San Francisco', + }, + }, + required: ['location'], + }, + strict: true, + }, + { + type: 'function', + name: 'send_email', + description: 'Send an email', + parameters: { + type: 'object', + properties: { + to: { + type: 'string', + description: 'The email address of the recipient', + }, + body: { + type: 'string', + description: 'The body of the email', + }, + }, + required: ['to', 'body'], + }, + strict: true, + }, + ]; + + const input: OpenAI.Responses.ResponseInput = [ + { + role: 'system', + content: 'You are a helpful assistant providing weather updates.', + }, + { + role: 'user', + content: 'What is the weather in Paris and Bogotá?', + }, + ]; + const response = await client.responses.create({ + model, + input, + tools, + parallel_tool_calls: true, + }); + + expect(response.output_text).toBe(''); + expect(response.output).toHaveLength(3); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['tool_call'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'system', + parts: [{ type: 'text', content: undefined }], + }, + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + 'id': 'fc_12345xyz', + 'call_id': 'call_12345xyz', + 'type': 'tool_call', + 'name': 'get_weather', + 'arguments': undefined + }, + { + 'id': 'fc_67890abc', + 'call_id': 'call_67890abc', + 'type': 'tool_call', + 'name': 'get_weather', + 'arguments': undefined + }, + { + 'id': 'fc_99999def', + 'call_id': 'call_99999def', + 'type': 'tool_call', + 'name': 'send_email', + 'arguments': undefined + } + ], + finish_reason: 'tool_call', + }, + ]); + }); + + it('records custom tool calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'custom', + name: 'code_exec', + description: 'Executes arbitrary Python code.', + }, + ]; + + const response = await client.responses.create({ + model, + input: 'Use the code_exec tool to print hello world to the console.', + tools, + }); + + expect(response.output_text).toBe(''); + expect(response.output).toHaveLength(2); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['tool_call'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'ctc_6890e975e86c819c9338825b3e1994810694874912ae0ea6', + name: 'code_exec', + arguments: undefined, + call_id: 'call_aGiFQkRWSWAIsMQ19fKqxUgb', + }, + ], + finish_reason: 'tool_call', + }, + ]); + }); + + it('records file search calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'file_search', + vector_store_ids: [''], + max_num_results: 2, + }, + ]; + + const response = await client.responses.create({ + model, + input: 'What is deep research by OpenAI?', + tools, + include: ['file_search_call.results'], + }); + + expect(response.output_text).toEqual( + expect.stringContaining('Deep research') + ); + expect(response.output).toHaveLength(2); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'fs_67c09ccea8c48191ade9367e3ba71515', + name: 'file_search_call', + arguments: undefined, + }, + { + type: 'tool_call_response', + id: 'fs_67c09ccea8c48191ade9367e3ba71515', + response: undefined, + }, + { + type: 'text', + content: undefined, + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('records web search calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'web_search', + }, + ]; + + const response = await client.responses.create({ + model, + input: 'What was a positive news story from today?', + tools, + include: ['web_search_call.action.sources'] + }); + + expect(response.output_text).toContain('On March 6, 2025, several news'); + expect(response.output).toHaveLength(2); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toMatchObject({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + }); + expect(spans[0].attributes[ATTR_GEN_AI_RESPONSE_FINISH_REASONS]).toEqual(['stop']); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'ws_67c9fa0502748190b7dd390736892e100be649c1a5ff9609', + name: 'web_search_call', + arguments: undefined, + }, + { + type: 'text', + content: undefined, + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('records computer use calls', async () => { + const computerModel = 'computer-use-preview'; + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'computer_use_preview', + display_width: 1024, + display_height: 768, + environment: 'browser', + }, + ]; + + const response = await client.responses.create({ + model: computerModel, + tools, + input: [ + { + role: 'user', + content: [ + { + type: 'input_text', + text: 'Check the latest OpenAI news on bing.com.', + }, + ], + }, + ], + reasoning: { summary: 'concise' }, + truncation: 'auto', + }); + + expect(response.output_text).toBe(''); + expect(response.output).toHaveLength(2); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: computerModel, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'computer-use-preview-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['tool_call'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: computerModel, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'computer-use-preview-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: computerModel, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'computer-use-preview-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: computerModel, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'computer-use-preview-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'reasoning', + text: undefined, + }, + { + type: 'tool_call', + id: 'cu_67cc...', + name: 'computer_call', + arguments: undefined, + call_id: 'call_zw3...', + }, + ], + finish_reason: 'tool_call', + }, + ]); + }); + + it('records code interpreter calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'code_interpreter', + container: 'cfile_682e0e8a43c88191a7978f477a09bdf5', + }, + ]; + + const response = await client.responses.create({ + model, + tools, + instructions: null, + input: 'Plot and analyse the histogram of the RGB channels for the uploaded image.', + }); + + expect(response.output_text).toContain('Here is the histogram of the RGB channels'); + expect(response.output).toHaveLength(1); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'text', + content: undefined, + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('records image generation call', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'image_generation', + }, + ]; + + const response = await client.responses.create({ + model, + tools, + input: [ + { + role: 'user', + content: [ + { + type: 'input_text', + text: 'Now make it look realistic', + }, + ], + }, + // @ts-expect-error: types don't match documentation + { + type: 'image_generation_call', + id: 'ig_123', + }, + ], + }); + + expect(response.output_text).toBe(''); + expect(response.output).toHaveLength(1); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + }, + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'ig_123', + name: 'image_generation_call', + }, + { + type: 'tool_call_response', + id: 'ig_123', + response: undefined, + }, + ], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'ig_124', + name: 'image_generation_call', + }, + { + type: 'tool_call_response', + id: 'ig_124', + response: undefined, + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('records mcp calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'mcp', + server_label: 'dmcp', + server_description: 'A Dungeons and Dragons MCP server to assist with dice rolling.', + server_url: 'https://dmcp-server.deno.dev/sse', + require_approval: 'never', + allowed_tools: ['roll'], + }, + ]; + + const response = await client.responses.create({ + model, + tools, + input: + [ + { + 'role': 'user', + 'content': 'Roll 2d4+1' + }, + { + 'id': 'mcpl_68a6102a4968819c8177b05584dd627b0679e572a900e618', + 'type': 'mcp_list_tools', + 'server_label': 'dmcp', + 'tools': [ + { + 'annotations': null, + 'description': 'Given a string of text describing a dice roll...', + 'input_schema': { + '$schema': 'https://json-schema.org/draft/2020-12/schema', + 'type': 'object', + 'properties': { + 'diceRollExpression': { + 'type': 'string' + } + }, + 'required': ['diceRollExpression'], + 'additionalProperties': false + }, + 'name': 'roll' + } + ] + }, + { + 'id': 'mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339', + 'type': 'mcp_approval_request', + 'arguments': '{"diceRollExpression":"2d4 + 1"}', + 'name': 'roll', + 'server_label': 'dmcp' + }, + { + 'type': 'mcp_approval_response', + 'approve': true, + 'approval_request_id': 'mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339' + } + ], + }); + + expect(response.output_text).toBe(''); + expect(response.output).toHaveLength(1); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + }, + { + role: 'assistant', + parts: [ + { + type: 'tool_call_response', + id: 'mcpl_68a6102a4968819c8177b05584dd627b0679e572a900e618', + response: undefined, + server: 'dmcp', + }, + ], + }, + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339', + name: 'mcp_approval_request', + arguments: undefined, + server: 'dmcp', + }, + ], + }, + { + role: 'user', + parts: [ + { + type: 'tool_call_response', + response: undefined, + }, + ], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'mcp_68a6102d8948819c9b1490d36d5ffa4a0679e572a900e618', + name: 'roll', + arguments: undefined, + server: 'dmcp', + }, + { + type: 'tool_call_response', + id: 'mcp_68a6102d8948819c9b1490d36d5ffa4a0679e572a900e618', + response: undefined, + server: 'dmcp', + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('handles connection errors without crashing', async () => { + expect( + new OpenAI({ + baseURL: 'http://localhost:9999/v5', + apiKey, + }).responses.create({ + model, + input, + }) + ).rejects.toThrow(OpenAI.APIConnectionError); + + await new Promise(resolve => setTimeout(resolve, 2000)); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'localhost', + [ATTR_SERVER_PORT]: 9999, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_ERROR_TYPE]: 'APIConnectionError', + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + expect(tokenUsage).toHaveLength(0); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'localhost', + [ATTR_SERVER_PORT]: 9999, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_ERROR_TYPE]: 'APIConnectionError', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(1); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + }, + ]); + }); + }); + + describe('responses with content capture', function () { + this.beforeEach(() => { + contentCaptureInstrumentation.enable(); + }); + this.afterEach(() => { + contentCaptureInstrumentation.disable(); + }); + + it('adds genai conventions', async () => { + const response = await client.responses.create({ + model, + input, + }); + + expect(response.output_text).toEqual('Atlantic Ocean.'); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 22, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 3, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 22, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 3, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: 'Answer in up to 3 words: Which ocean contains Bouvet Island?' }], + } + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [{ type: 'text', content: 'Atlantic Ocean.' }], + finish_reason: 'stop' + } + + ]) + }); + + it('records function calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'function', + name: 'get_weather', + description: 'Get the current weather in a given location', + parameters: { + type: 'object', + properties: { + location: { + type: 'string', + description: 'The city, e.g. San Francisco', + }, + }, + required: ['location'], + }, + strict: true, + }, + { + type: 'function', + name: 'send_email', + description: 'Send an email', + parameters: { + type: 'object', + properties: { + to: { + type: 'string', + description: 'The email address of the recipient', + }, + body: { + type: 'string', + description: 'The body of the email', + }, + }, + required: ['to', 'body'], + }, + strict: true, + }, + ]; + + const input: OpenAI.Responses.ResponseInput = [ + { + role: 'system', + content: 'You are a helpful assistant providing weather updates.', + }, + { + role: 'user', + content: 'What is the weather in Paris and Bogotá?', + }, + ]; + const response = await client.responses.create({ + model, + input, + tools, + parallel_tool_calls: true, + }); + + expect(response.output_text).toBe(''); + expect(response.output).toHaveLength(3); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['tool_call'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'system', + parts: [ + { + type: 'text', + content: 'You are a helpful assistant providing weather updates.', + }, + ], + }, + { + role: 'user', + parts: [ + { + type: 'text', + content: 'What is the weather in Paris and Bogotá?', + }, + ], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + 'id': 'fc_12345xyz', + 'call_id': 'call_12345xyz', + 'type': 'tool_call', + 'name': 'get_weather', + 'arguments': '{"location":"Paris, France"}' + }, + { + 'id': 'fc_67890abc', + 'call_id': 'call_67890abc', + 'type': 'tool_call', + 'name': 'get_weather', + 'arguments': '{"location":"Bogotá, Colombia"}' + }, + { + 'id': 'fc_99999def', + 'call_id': 'call_99999def', + 'type': 'tool_call', + 'name': 'send_email', + 'arguments': '{"to":"bob@email.com","body":"Hi bob"}' + } + ], + finish_reason: 'tool_call', + }, + ]); + }); + + it('records custom tool calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'custom', + name: 'code_exec', + description: 'Executes arbitrary Python code.', + }, + ]; + + const response = await client.responses.create({ + model, + input: 'Use the code_exec tool to print hello world to the console.', + tools, + }); + + expect(response.output_text).toBe(''); + expect(response.output).toHaveLength(2); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['tool_call'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [ + { + type: 'text', + content: 'Use the code_exec tool to print hello world to the console.', + }, + ], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'ctc_6890e975e86c819c9338825b3e1994810694874912ae0ea6', + name: 'code_exec', + arguments: 'print("hello world")', + call_id: 'call_aGiFQkRWSWAIsMQ19fKqxUgb', + }, + ], + finish_reason: 'tool_call', + }, + ]); + }); + + it('records file search calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'file_search', + vector_store_ids: [''], + max_num_results: 2, + }, + ]; + + const response = await client.responses.create({ + model, + input: 'What is deep research by OpenAI?', + tools, + include: ['file_search_call.results'], + }); + + expect(response.output_text).toEqual( + expect.stringContaining('Deep research') + ); + expect(response.output).toHaveLength(2); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [ + { + type: 'text', + content: 'What is deep research by OpenAI?', + }, + ], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'fs_67c09ccea8c48191ade9367e3ba71515', + name: 'file_search_call', + arguments: ['What is deep research?'], + }, + { + type: 'tool_call_response', + id: 'fs_67c09ccea8c48191ade9367e3ba71515', + response: expect.objectContaining({ + id: 'file-2dtbBZdjtDKS8eqWxqbgDi', + filename: 'deep_research_blog.pdf', + score: 0.95, + text: expect.stringContaining('Lorem ipsum dolor'), + }), + }, + { + type: 'text', + content: expect.stringContaining('Deep research'), + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('records web search calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'web_search', + }, + ]; + + const response = await client.responses.create({ + model, + input: 'What was a positive news story from today?', + tools, + include: ['web_search_call.action.sources'] + }); + + expect(response.output_text).toContain('On March 6, 2025, several news'); + expect(response.output).toHaveLength(2); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'] + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [ + { + type: 'text', + content: 'What was a positive news story from today?', + }, + ], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'ws_67c9fa0502748190b7dd390736892e100be649c1a5ff9609', + name: 'web_search_call', + arguments: 'search', + }, + { + type: 'text', + content: expect.stringContaining('On March 6, 2025'), + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('records computer use calls', async () => { + const computerModel = 'computer-use-preview'; + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'computer_use_preview', + display_width: 1024, + display_height: 768, + environment: 'browser', + }, + ]; + + const response = await client.responses.create({ + model: computerModel, + tools, + input: [ + { + role: 'user', + content: [ + { + type: 'input_text', + text: 'Check the latest OpenAI news on bing.com.', + }, + ], + }, + ], + reasoning: { summary: 'concise' }, + truncation: 'auto', + }); + + expect(response.output_text).toBe(''); + expect(response.output).toHaveLength(2); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: computerModel, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'computer-use-preview-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['tool_call'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: computerModel, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'computer-use-preview-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: computerModel, + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'computer-use-preview-2025-04-14', + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: computerModel, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'computer-use-preview-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [ + { + type: 'text', + content: 'Check the latest OpenAI news on bing.com.', + }, + ], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'reasoning', + text: 'Clicking on the browser address bar.', + }, + { + type: 'tool_call', + id: 'cu_67cc...', + name: 'computer_call', + arguments: { + type: 'click', + button: 'left', + x: 156, + y: 50, + }, + call_id: 'call_zw3...', + }, + ], + finish_reason: 'tool_call', + }, + ]); + }); + + it('records code interpreter calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'code_interpreter', + container: 'cfile_682e0e8a43c88191a7978f477a09bdf5', + }, + ]; + + const response = await client.responses.create({ + model, + tools, + instructions: null, + input: 'Plot and analyse the histogram of the RGB channels for the uploaded image.', + }); + + expect(response.output_text).toContain('Here is the histogram of the RGB channels'); + expect(response.output).toHaveLength(1); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [ + { + type: 'text', + content: 'Plot and analyse the histogram of the RGB channels for the uploaded image.', + }, + ], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'text', + content: expect.stringContaining('Here is the histogram of the RGB channels for the uploaded image'), + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('records image generation call', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'image_generation', + }, + ]; + + const response = await client.responses.create({ + model, + tools, + input: [ + { + role: 'user', + content: [ + { + type: 'input_text', + text: 'Now make it look realistic', + }, + ], + }, + // @ts-expect-error: types don't match documentation + { + type: 'image_generation_call', + id: 'ig_123', + }, + ], + }); + + expect(response.output_text).toBe(''); + expect(response.output).toHaveLength(1); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [ + { + type: 'text', + content: 'Now make it look realistic', + }, + ], + }, + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'ig_123', + name: 'image_generation_call', + }, + { + type: 'tool_call_response', + id: 'ig_123', + response: undefined, + }, + ], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'ig_124', + name: 'image_generation_call', + }, + { + type: 'tool_call_response', + id: 'ig_124', + response: '...', + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('records mcp calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'mcp', + server_label: 'dmcp', + server_description: 'A Dungeons and Dragons MCP server to assist with dice rolling.', + server_url: 'https://dmcp-server.deno.dev/sse', + require_approval: 'never', + allowed_tools: ['roll'], + }, + ]; + + const response = await client.responses.create({ + model, + tools, + input: + [ + { + 'role': 'user', + 'content': 'Roll 2d4+1' + }, + { + 'id': 'mcpl_68a6102a4968819c8177b05584dd627b0679e572a900e618', + 'type': 'mcp_list_tools', + 'server_label': 'dmcp', + 'tools': [ + { + 'annotations': null, + 'description': 'Given a string of text describing a dice roll...', + 'input_schema': { + '$schema': 'https://json-schema.org/draft/2020-12/schema', + 'type': 'object', + 'properties': { + 'diceRollExpression': { + 'type': 'string' + } + }, + 'required': ['diceRollExpression'], + 'additionalProperties': false + }, + 'name': 'roll' + } + ] + }, + { + 'id': 'mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339', + 'type': 'mcp_approval_request', + 'arguments': '{"diceRollExpression":"2d4 + 1"}', + 'name': 'roll', + 'server_label': 'dmcp' + }, + { + 'type': 'mcp_approval_response', + 'approve': true, + 'approval_request_id': 'mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339' + } + ], + }); + + expect(response.output_text).toBe(''); + expect(response.output).toHaveLength(1); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [ + { + type: 'text', + content: 'Roll 2d4+1', + }, + ], + }, + { + role: 'assistant', + parts: [ + { + type: 'tool_call_response', + id: 'mcpl_68a6102a4968819c8177b05584dd627b0679e572a900e618', + response: expect.arrayContaining([ + expect.objectContaining({ + name: 'roll', + description: 'Given a string of text describing a dice roll...', + }), + ]), + server: 'dmcp', + }, + ], + }, + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339', + name: 'mcp_approval_request: roll', + arguments: '{"diceRollExpression":"2d4 + 1"}', + server: 'dmcp', + }, + ], + }, + { + role: 'user', + parts: [ + { + type: 'tool_call_response', + response: true, + }, + ], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'mcp_68a6102d8948819c9b1490d36d5ffa4a0679e572a900e618', + name: 'roll', + arguments: 'roll({"diceRollExpression":"2d4 + 1"})', + server: 'dmcp', + }, + { + type: 'tool_call_response', + id: 'mcp_68a6102d8948819c9b1490d36d5ffa4a0679e572a900e618', + response: '4', + server: 'dmcp', + }, + ], + finish_reason: 'stop', + }, + ]); + }); + }); + + describe('streaming responses', function () { + this.beforeEach(() => { + instrumentation.enable(); + }); + this.afterEach(() => { + instrumentation.disable(); + }); + + it('adds genai conventions', async () => { + const stream = client.responses.stream({ + model, + input, + }); + let content = ''; + for await (const part of stream) { + content += part.type === 'response.output_text.delta' ? part.delta : ''; + } + expect(content).toEqual('Atlantic Ocean.'); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 22, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 3, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 22, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 3, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + } + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [{ type: 'text', content: undefined }], + finish_reason: 'stop' + } + ]); + }); + + it('records function calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'function', + name: 'get_weather', + description: 'Get the current weather in a given location', + parameters: { + type: 'object', + properties: { + location: { + type: 'string', + description: 'The city, e.g. San Francisco', + }, + }, + required: ['location'], + }, + strict: true, + }, + { + type: 'function', + name: 'send_email', + description: 'Send an email', + parameters: { + type: 'object', + properties: { + to: { + type: 'string', + description: 'The email address of the recipient', + }, + body: { + type: 'string', + description: 'The body of the email', + }, + }, + required: ['to', 'body'], + }, + strict: true, + }, + ]; + + const input: OpenAI.Responses.ResponseInput = [ + { + role: 'system', + content: 'You are a helpful assistant providing weather updates.', + }, + { + role: 'user', + content: 'What is the weather in Paris and Bogotá?', + }, + ]; + const stream = client.responses.stream({ + model, + input, + tools, + parallel_tool_calls: true, + }); + + const events: Array = []; + for await (const event of stream) { + events.push(event); + } + + const completedEvent = events.find(event => event.type === 'response.completed'); + const doneEvents = events.filter(event => event.type === 'response.output_item.done'); + + expect(completedEvent?.response.output).toHaveLength(3); + expect(doneEvents).toHaveLength(3); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['tool_call'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(1 + doneEvents.length); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'system', + parts: [{ type: 'text', content: undefined }], + }, + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + }, + ]); + + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'fc_12345xyz', + name: 'get_weather', + arguments: undefined, + call_id: 'call_12345xyz', + }, + ], + finish_reason: 'tool_call', + }, + ]); + expect(logs[2].spanContext).toEqual(spanCtx); + expect(logs[2].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[2].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[2].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'fc_67890abc', + name: 'get_weather', + arguments: undefined, + call_id: 'call_67890abc', + }, + ], + finish_reason: 'tool_call', + }, + ]); + expect(logs[3].spanContext).toEqual(spanCtx); + expect(logs[3].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[3].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[3].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'fc_99999def', + name: 'send_email', + arguments: undefined, + call_id: 'call_99999def', + }, + ], + finish_reason: 'tool_call', + }, + ]); + }); + + it('records custom tool calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'custom', + name: 'code_exec', + description: 'Executes arbitrary Python code.', + }, + ]; + + const stream = client.responses.stream({ + model, + input: 'Use the code_exec tool to print hello world to the console.', + tools, + }); + + const events: Array = []; + for await (const event of stream) { + events.push(event); + } + + const completedEvent = events.find(event => event.type === 'response.completed'); + const doneEvents = events.filter(event => event.type === 'response.output_item.done'); + + expect(completedEvent?.response.output).toHaveLength(2); + expect(doneEvents).toHaveLength(2); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['tool_call'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(1 + doneEvents.length); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + }, + ]); + + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [], + finish_reason: 'stop', + }, + ]); + expect(logs[2].spanContext).toEqual(spanCtx); + expect(logs[2].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[2].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[2].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'ctc_6890e975e86c819c9338825b3e1994810694874912ae0ea6', + name: 'code_exec', + arguments: undefined, + call_id: 'call_aGiFQkRWSWAIsMQ19fKqxUgb', + }, + ], + finish_reason: 'tool_call', + }, + ]); + }); + + it('records file search calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'file_search', + vector_store_ids: [''], + max_num_results: 2, + }, + ]; + + const stream = client.responses.stream({ + model, + input: 'What is deep research by OpenAI?', + tools, + include: ['file_search_call.results'], + }); + + const events: Array = []; + for await (const event of stream) { + events.push(event); + } + + const completedEvent = events.find(event => event.type === 'response.completed'); + const doneEvents = events.filter(event => event.type === 'response.output_item.done'); + + expect(completedEvent?.response.output).toHaveLength(2); + expect(doneEvents).toHaveLength(2); + + const messageEvent = doneEvents.find(event => event.item.type === 'message'); + expect(messageEvent).toBeDefined(); + expect(messageEvent?.item.content[0].text).toEqual( + expect.stringContaining('Deep research') + ); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(1 + doneEvents.length); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + }, + ]); + + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'fs_67c09ccea8c48191ade9367e3ba71515', + name: 'file_search_call', + arguments: undefined, + }, + { + type: 'tool_call_response', + id: 'fs_67c09ccea8c48191ade9367e3ba71515', + response: undefined, + }, + ], + finish_reason: 'stop', + }, + ]); + expect(logs[2].spanContext).toEqual(spanCtx); + expect(logs[2].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[2].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[2].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'text', + content: undefined, + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('records web search calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'web_search', + }, + ]; + + const stream = client.responses.stream({ + model, + input: 'What was a positive news story from today?', + tools, + include: ['web_search_call.action.sources'], + }); + + const events: Array = []; + for await (const event of stream) { + events.push(event); + } + + const completedEvent = events.find(event => event.type === 'response.completed'); + const doneEvents = events.filter(event => event.type === 'response.output_item.done'); + + expect(completedEvent?.response.output).toHaveLength(2); + expect(doneEvents).toHaveLength(2); + + const messageEvent = doneEvents.find(event => event.item.type === 'message'); + expect(messageEvent).toBeDefined(); + expect(messageEvent?.item.content[0].text).toContain('On March 6, 2025, several news'); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toMatchObject({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + }); + expect(spans[0].attributes[ATTR_GEN_AI_RESPONSE_FINISH_REASONS]).toEqual(['stop']); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(1 + doneEvents.length); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + }, + ]); + + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'ws_67c9fa0502748190b7dd390736892e100be649c1a5ff9609', + name: 'web_search_call', + arguments: undefined, + }, + ], + finish_reason: 'tool_call', + }, + ]); + expect(logs[2].spanContext).toEqual(spanCtx); + expect(logs[2].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[2].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[2].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'text', + content: undefined, + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('records computer use calls', async () => { + const computerModel = 'computer-use-preview'; + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'computer_use_preview', + display_width: 1024, + display_height: 768, + environment: 'browser', + }, + ]; + + const stream = client.responses.stream({ + model: computerModel, + tools, + input: [ + { + role: 'user', + content: [ + { + type: 'input_text', + text: 'Check the latest OpenAI news on bing.com.', + }, + ], + }, + ], + reasoning: { summary: 'concise' }, + truncation: 'auto', + }); + + const events: Array = []; + for await (const event of stream) { + events.push(event); + } + + const completedEvent = events.find(event => event.type === 'response.completed'); + const doneEvents = events.filter(event => event.type === 'response.output_item.done'); + + expect(completedEvent?.response.output).toHaveLength(2); + expect(doneEvents).toHaveLength(2); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: computerModel, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'computer-use-preview-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['tool_call'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: computerModel, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'computer-use-preview-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: computerModel, + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'computer-use-preview-2025-04-14', + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: computerModel, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'computer-use-preview-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(1 + doneEvents.length); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + }, + ]); + + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'reasoning', + text: undefined, + }, + ], + finish_reason: 'stop', + }, + ]); + expect(logs[2].spanContext).toEqual(spanCtx); + expect(logs[2].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[2].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[2].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'cu_67cc...', + name: 'computer_call', + arguments: undefined, + call_id: 'call_zw3...', + }, + ], + finish_reason: 'tool_call', + }, + ]); + }); + + it('records code interpreter calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'code_interpreter', + container: 'cfile_682e0e8a43c88191a7978f477a09bdf5', + }, + ]; + + const stream = client.responses.stream({ + model, + tools, + instructions: null, + input: 'Plot and analyse the histogram of the RGB channels for the uploaded image.', + }); + + const events: Array = []; + for await (const event of stream) { + events.push(event); + } + + const completedEvent = events.find(event => event.type === 'response.completed'); + const doneEvents = events.filter(event => event.type === 'response.output_item.done'); + + expect(completedEvent?.response.output).toHaveLength(1); + expect(doneEvents).toHaveLength(1); + expect(doneEvents[0].item.content[0].text).toContain('Here is the histogram of the RGB channels'); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'text', + content: undefined, + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('records image generation call', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'image_generation', + }, + ]; + + const stream = client.responses.stream({ + model, + tools, + input: [ + { + role: 'user', + content: [ + { + type: 'input_text', + text: 'Now make it look realistic', + }, + ], + }, + // @ts-expect-error: types don't match documentation + { + type: 'image_generation_call', + id: 'ig_123', + }, + ], + }); + + const events: Array = []; + for await (const event of stream) { + events.push(event); + } + + const completedEvent = events.find(event => event.type === 'response.completed'); + const doneEvents = events.filter(event => event.type === 'response.output_item.done'); + + expect(completedEvent?.response.output).toHaveLength(1); + expect(doneEvents).toHaveLength(1); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [ + { + type: 'text', + content: undefined, + }, + ], + }, + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'ig_123', + name: 'image_generation_call', + arguments: undefined, + }, + { + type: 'tool_call_response', + id: 'ig_123', + response: undefined, + }, + ], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'ig_124', + name: 'image_generation_call', + }, + { + type: 'tool_call_response', + id: 'ig_124', + response: undefined, + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('records mcp calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'mcp', + server_label: 'dmcp', + server_description: 'A Dungeons and Dragons MCP server to assist with dice rolling.', + server_url: 'https://dmcp-server.deno.dev/sse', + require_approval: 'never', + allowed_tools: ['roll'], + }, + ]; + + const stream = client.responses.stream({ + model, + tools, + input: [ + { + role: 'user', + content: 'Roll 2d4+1' + }, + { + id: 'mcpl_68a6102a4968819c8177b05584dd627b0679e572a900e618', + type: 'mcp_list_tools', + server_label: 'dmcp', + tools: [ + { + annotations: null, + description: 'Given a string of text describing a dice roll...', + input_schema: { + '$schema': 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + properties: { + diceRollExpression: { + type: 'string' + } + }, + required: ['diceRollExpression'], + additionalProperties: false + }, + name: 'roll' + } + ] + }, + { + id: 'mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339', + type: 'mcp_approval_request', + arguments: '{"diceRollExpression":"2d4 + 1"}', + name: 'roll', + server_label: 'dmcp' + }, + { + type: 'mcp_approval_response', + approve: true, + approval_request_id: 'mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339' + } + ], + }); + + const events: Array = []; + for await (const event of stream) { + events.push(event); + } + + const completedEvent = events.find(event => event.type === 'response.completed'); + const doneEvents = events.filter(event => event.type === 'response.output_item.done'); + + expect(completedEvent?.response.output).toHaveLength(1); + expect(doneEvents).toHaveLength(1); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(1 + doneEvents.length); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + }, + { + role: 'assistant', + parts: [ + { + type: 'tool_call_response', + id: 'mcpl_68a6102a4968819c8177b05584dd627b0679e572a900e618', + response: undefined, + server: 'dmcp', + }, + ], + }, + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339', + name: 'mcp_approval_request', + arguments: undefined, + server: 'dmcp', + }, + ], + }, + { + role: 'user', + parts: [ + { + type: 'tool_call_response', + response: undefined, + }, + ], + }, + ]); + + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'mcp_68a6102d8948819c9b1490d36d5ffa4a0679e572a900e618', + name: 'roll', + arguments: undefined, + server: 'dmcp', + }, + { + type: 'tool_call_response', + id: 'mcp_68a6102d8948819c9b1490d36d5ffa4a0679e572a900e618', + response: undefined, + server: 'dmcp', + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('does not misbehave with double iteration', async () => { + const stream = client.responses.stream({ + model, + input, + }); + let content = ''; + for await (const event of stream) { + if (event.type === 'response.output_text.delta') { + content += event.delta; + } + } + expect(async () => { + for await (const event of stream) { + if (event.type === 'response.output_text.delta') { + content += event.delta; + } + } + }).rejects.toThrow(); + expect(content).toEqual('South Atlantic Ocean.'); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 22, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 4, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: undefined }], + } + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [{ type: 'text', content: undefined }], + finish_reason: 'stop' + } + ]); + }); + }); + + describe('streaming responses with content capture', function () { + this.beforeEach(() => { + contentCaptureInstrumentation.enable(); + }); + this.afterEach(() => { + contentCaptureInstrumentation.disable(); + }); + + it('adds genai conventions', async () => { + const stream = client.responses.stream({ + model, + input, + }); + let content = ''; + for await (const part of stream) { + content += part.type === 'response.output_text.delta' ? part.delta : ''; + } + expect(content).toEqual('Atlantic Ocean.'); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 22, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 3, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 22, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 3, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: 'Answer in up to 3 words: Which ocean contains Bouvet Island?' }], + } + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [{ type: 'text', content: 'Atlantic Ocean.' }], + finish_reason: 'stop' + } + ]); + }); + + it('records function calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'function', + name: 'get_weather', + description: 'Get the current weather in a given location', + parameters: { + type: 'object', + properties: { + location: { + type: 'string', + description: 'The city, e.g. San Francisco', + }, + }, + required: ['location'], + }, + strict: true, + }, + { + type: 'function', + name: 'send_email', + description: 'Send an email', + parameters: { + type: 'object', + properties: { + to: { + type: 'string', + description: 'The email address of the recipient', + }, + body: { + type: 'string', + description: 'The body of the email', + }, + }, + required: ['to', 'body'], + }, + strict: true, + }, + ]; + + const input: OpenAI.Responses.ResponseInput = [ + { + role: 'system', + content: 'You are a helpful assistant providing weather updates.', + }, + { + role: 'user', + content: 'What is the weather in Paris and Bogotá?', + }, + ]; + const stream = client.responses.stream({ + model, + input, + tools, + parallel_tool_calls: true, + }); + + const events: Array = []; + for await (const event of stream) { + events.push(event); + } + + const completedEvent = events.find(event => event.type === 'response.completed'); + const doneEvents = events.filter(event => event.type === 'response.output_item.done'); + + expect(completedEvent?.response.output).toHaveLength(3); + expect(doneEvents).toHaveLength(3); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['tool_call'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(1 + doneEvents.length); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'system', + parts: [{ type: 'text', content: 'You are a helpful assistant providing weather updates.' }], + }, + { + role: 'user', + parts: [{ type: 'text', content: 'What is the weather in Paris and Bogotá?' }], + }, + ]); + + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'fc_12345xyz', + name: 'get_weather', + arguments: '{"location":"Paris, France"}', + call_id: 'call_12345xyz', + }, + ], + finish_reason: 'tool_call', + }, + ]); + expect(logs[2].spanContext).toEqual(spanCtx); + expect(logs[2].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[2].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[2].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'fc_67890abc', + name: 'get_weather', + arguments: '{"location":"Bogotá, Colombia"}', + call_id: 'call_67890abc', + }, + ], + finish_reason: 'tool_call', + }, + ]); + expect(logs[3].spanContext).toEqual(spanCtx); + expect(logs[3].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[3].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[3].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'fc_99999def', + name: 'send_email', + arguments: '{"to":"bob@email.com","body":"Hi bob"}', + call_id: 'call_99999def', + }, + ], + finish_reason: 'tool_call', + }, + ]); + }); + + it('records custom tool calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'custom', + name: 'code_exec', + description: 'Executes arbitrary Python code.', + }, + ]; + + const stream = client.responses.stream({ + model, + input: 'Use the code_exec tool to print hello world to the console.', + tools, + }); + + const events: Array = []; + for await (const event of stream) { + events.push(event); + } + + const completedEvent = events.find(event => event.type === 'response.completed'); + const doneEvents = events.filter(event => event.type === 'response.output_item.done'); + + expect(completedEvent?.response.output).toHaveLength(2); + expect(doneEvents).toHaveLength(2); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['tool_call'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(1 + doneEvents.length); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: 'Use the code_exec tool to print hello world to the console.' }], + }, + ]); + + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [], + finish_reason: 'stop', + }, + ]); + expect(logs[2].spanContext).toEqual(spanCtx); + expect(logs[2].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[2].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[2].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'ctc_6890e975e86c819c9338825b3e1994810694874912ae0ea6', + name: 'code_exec', + arguments: 'print("hello world")', + call_id: 'call_aGiFQkRWSWAIsMQ19fKqxUgb', + }, + ], + finish_reason: 'tool_call', + }, + ]); + }); + + it('records file search calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'file_search', + vector_store_ids: [''], + max_num_results: 2, + }, + ]; + + const stream = client.responses.stream({ + model, + input: 'What is deep research by OpenAI?', + tools, + include: ['file_search_call.results'], + }); + + const events: Array = []; + for await (const event of stream) { + events.push(event); + } + + const completedEvent = events.find(event => event.type === 'response.completed'); + const doneEvents = events.filter(event => event.type === 'response.output_item.done'); + + expect(completedEvent?.response.output).toHaveLength(2); + expect(doneEvents).toHaveLength(2); + + const messageEvent = doneEvents.find(event => event.item.type === 'message'); + expect(messageEvent).toBeDefined(); + expect(messageEvent?.item.content[0].text).toEqual( + expect.stringContaining('Deep research') + ); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(1 + doneEvents.length); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: 'What is deep research by OpenAI?' }], + }, + ]); + + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'fs_67c09ccea8c48191ade9367e3ba71515', + name: 'file_search_call', + arguments: ['What is deep research?'], + }, + { + type: 'tool_call_response', + id: 'fs_67c09ccea8c48191ade9367e3ba71515', + response: { filename: 'deep_research_blog.pdf', id: 'file-2dtbBZdjtDKS8eqWxqbgDi', score: 0.95, text: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed euismod, nisl eget aliquam aliquet, nunc nisl aliquet nisl, eget aliquam nisl nisl eget nisl. Sed euismod, nisl eget aliquam aliquet, nunc nisl aliquet nisl, eget aliquam nisl nisl eget nisl.' }, + }, + ], + finish_reason: 'stop', + }, + ]); + expect(logs[2].spanContext).toEqual(spanCtx); + expect(logs[2].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[2].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[2].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'text', + content: expect.stringContaining('Deep research'), + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('records web search calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'web_search', + }, + ]; + + const stream = client.responses.stream({ + model, + input: 'What was a positive news story from today?', + tools, + include: ['web_search_call.action.sources'], + }); + + const events: Array = []; + for await (const event of stream) { + events.push(event); + } + + const completedEvent = events.find(event => event.type === 'response.completed'); + const doneEvents = events.filter(event => event.type === 'response.output_item.done'); + + expect(completedEvent?.response.output).toHaveLength(2); + expect(doneEvents).toHaveLength(2); + + const messageEvent = doneEvents.find(event => event.item.type === 'message'); + expect(messageEvent).toBeDefined(); + expect(messageEvent?.item.content[0].text).toContain('On March 6, 2025, several news'); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toMatchObject({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + }); + expect(spans[0].attributes[ATTR_GEN_AI_RESPONSE_FINISH_REASONS]).toEqual(['stop']); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(1 + doneEvents.length); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: 'What was a positive news story from today?' }], + }, + ]); + + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'ws_67c9fa0502748190b7dd390736892e100be649c1a5ff9609', + name: 'web_search_call', + arguments: 'search', + }, + ], + finish_reason: 'tool_call', + }, + ]); + expect(logs[2].spanContext).toEqual(spanCtx); + expect(logs[2].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[2].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[2].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'text', + content: 'On March 6, 2025, several news...', + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('records computer use calls', async () => { + const computerModel = 'computer-use-preview'; + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'computer_use_preview', + display_width: 1024, + display_height: 768, + environment: 'browser', + }, + ]; + + const stream = client.responses.stream({ + model: computerModel, + tools, + input: [ + { + role: 'user', + content: [ + { + type: 'input_text', + text: 'Check the latest OpenAI news on bing.com.', + }, + ], + }, + ], + reasoning: { summary: 'concise' }, + truncation: 'auto', + }); + + const events: Array = []; + for await (const event of stream) { + events.push(event); + } + + const completedEvent = events.find(event => event.type === 'response.completed'); + const doneEvents = events.filter(event => event.type === 'response.output_item.done'); + + expect(completedEvent?.response.output).toHaveLength(2); + expect(doneEvents).toHaveLength(2); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: computerModel, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'computer-use-preview-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['tool_call'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: computerModel, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'computer-use-preview-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: computerModel, + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'computer-use-preview-2025-04-14', + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: computerModel, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'computer-use-preview-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(1 + doneEvents.length); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: 'Check the latest OpenAI news on bing.com.' }], + }, + ]); + + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'reasoning', + text: 'Clicking on the browser address bar.', + }, + ], + finish_reason: 'stop', + }, + ]); + expect(logs[2].spanContext).toEqual(spanCtx); + expect(logs[2].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[2].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[2].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'cu_67cc...', + name: 'computer_call', + arguments: { button: 'left', type: 'click', x: 156, y: 50 }, + call_id: 'call_zw3...', + }, + ], + finish_reason: 'tool_call', + }, + ]); + }); + + it('records code interpreter calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'code_interpreter', + container: 'cfile_682e0e8a43c88191a7978f477a09bdf5', + }, + ]; + + const stream = client.responses.stream({ + model, + tools, + instructions: null, + input: 'Plot and analyse the histogram of the RGB channels for the uploaded image.', + }); + + const events: Array = []; + for await (const event of stream) { + events.push(event); + } + + const completedEvent = events.find(event => event.type === 'response.completed'); + const doneEvents = events.filter(event => event.type === 'response.output_item.done'); + + expect(completedEvent?.response.output).toHaveLength(1); + expect(doneEvents).toHaveLength(1); + expect(doneEvents[0].item.content[0].text).toContain('Here is the histogram of the RGB channels'); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: 'Plot and analyse the histogram of the RGB channels for the uploaded image.' }], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'text', + content: 'Here is the histogram of the RGB channels for the uploaded image. Each curve represents the distribution of pixel intensities for the red, green, and blue channels. Peaks toward the high end of the intensity scale (right-hand side) suggest a lot of brightness and strong warm tones, matching the orange and light background in the image. If you want a different style of histogram (e.g., overall intensity, or quantized color groups), let me know!', + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('records image generation call', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'image_generation', + }, + ]; + + const stream = client.responses.stream({ + model, + tools, + input: [ + { + role: 'user', + content: [ + { + type: 'input_text', + text: 'Now make it look realistic', + }, + ], + }, + // @ts-expect-error: types don't match documentation + { + type: 'image_generation_call', + id: 'ig_123', + }, + ], + }); + + const events: Array = []; + for await (const event of stream) { + events.push(event); + } + + const completedEvent = events.find(event => event.type === 'response.completed'); + const doneEvents = events.filter(event => event.type === 'response.output_item.done'); + + expect(completedEvent?.response.output).toHaveLength(1); + expect(doneEvents).toHaveLength(1); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [ + { + type: 'text', + content: 'Now make it look realistic', + }, + ], + }, + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'ig_123', + name: 'image_generation_call', + }, + { + type: 'tool_call_response', + id: 'ig_123', + response: undefined, + }, + ], + }, + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'ig_124', + name: 'image_generation_call', + }, + { + type: 'tool_call_response', + id: 'ig_124', + response: '...', + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('records mcp calls', async () => { + const tools: OpenAI.Responses.Tool[] = [ + { + type: 'mcp', + server_label: 'dmcp', + server_description: 'A Dungeons and Dragons MCP server to assist with dice rolling.', + server_url: 'https://dmcp-server.deno.dev/sse', + require_approval: 'never', + allowed_tools: ['roll'], + }, + ]; + + const stream = client.responses.stream({ + model, + tools, + input: [ + { + role: 'user', + content: 'Roll 2d4+1' + }, + { + id: 'mcpl_68a6102a4968819c8177b05584dd627b0679e572a900e618', + type: 'mcp_list_tools', + server_label: 'dmcp', + tools: [ + { + annotations: null, + description: 'Given a string of text describing a dice roll...', + input_schema: { + '$schema': 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + properties: { + diceRollExpression: { + type: 'string' + } + }, + required: ['diceRollExpression'], + additionalProperties: false + }, + name: 'roll' + } + ] + }, + { + id: 'mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339', + type: 'mcp_approval_request', + arguments: '{"diceRollExpression":"2d4 + 1"}', + name: 'roll', + server_label: 'dmcp' + }, + { + type: 'mcp_approval_response', + approve: true, + approval_request_id: 'mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339' + } + ], + }); + + const events: Array = []; + for await (const event of stream) { + events.push(event); + } + + const completedEvent = events.find(event => event.type === 'response.completed'); + const doneEvents = events.filter(event => event.type === 'response.output_item.done'); + + expect(completedEvent?.response.output).toHaveLength(1); + expect(doneEvents).toHaveLength(1); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 291, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 23, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + expect(tokenUsage[0].descriptor).toMatchObject({ + name: 'gen_ai.client.token.usage', + type: 'HISTOGRAM', + description: 'Measures number of input and output tokens used', + unit: '{token}', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(tokenUsage[0].dataPoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: 291, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_INPUT, + }, + }), + expect.objectContaining({ + value: expect.objectContaining({ + sum: 23, + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + [ATTR_GEN_AI_TOKEN_TYPE]: GEN_AI_TOKEN_TYPE_VALUE_OUTPUT, + }, + }), + ]) + ); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(operationDuration[0].dataPoints.length).toBe(1); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4.1-2025-04-14', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(1 + doneEvents.length); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: 'Roll 2d4+1' }], + }, + { + role: 'assistant', + parts: [ + { + type: 'tool_call_response', + id: 'mcpl_68a6102a4968819c8177b05584dd627b0679e572a900e618', + response: [{ annotations: null, description: 'Given a string of text describing a dice roll...', input_schema: { '$schema': 'https://json-schema.org/draft/2020-12/schema', type: 'object', properties: { diceRollExpression: { type: 'string' } }, required: ['diceRollExpression'], additionalProperties: false }, name: 'roll' }], + server: 'dmcp', + }, + ], + }, + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339', + name: 'mcp_approval_request: roll', + arguments: '{"diceRollExpression":"2d4 + 1"}', + server: 'dmcp', + }, + ], + }, + { + role: 'user', + parts: [ + { + type: 'tool_call_response', + response: true, + }, + ], + }, + ]); + + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [ + { + type: 'tool_call', + id: 'mcp_68a6102d8948819c9b1490d36d5ffa4a0679e572a900e618', + name: 'roll', + arguments: 'roll({"diceRollExpression":"2d4 + 1"})', + server: 'dmcp', + }, + { + type: 'tool_call_response', + id: 'mcp_68a6102d8948819c9b1490d36d5ffa4a0679e572a900e618', + response: '4', + server: 'dmcp', + }, + ], + finish_reason: 'stop', + }, + ]); + }); + + it('does not misbehave with double iteration', async () => { + const stream = client.responses.stream({ + model, + input, + }); + let content = ''; + for await (const event of stream) { + if (event.type === 'response.output_text.delta') { + content += event.delta; + } + } + expect(async () => { + for await (const event of stream) { + if (event.type === 'response.output_text.delta') { + content += event.delta; + } + } + }).rejects.toThrow(); + expect(content).toEqual('South Atlantic Ocean.'); + + const spans = getTestSpans(); + expect(spans.length).toBe(1); + expect(spans[0].attributes).toEqual({ + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + [ATTR_GEN_AI_RESPONSE_ID]: expect.stringMatching(/^resp_/), + [ATTR_GEN_AI_USAGE_INPUT_TOKENS]: 22, + [ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: 4, + [ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'], + }); + + await meterProvider.forceFlush(); + const [resourceMetrics] = metricExporter.getMetrics(); + expect(resourceMetrics.scopeMetrics.length).toBe(1); + const scopeMetrics = resourceMetrics.scopeMetrics[0]; + const tokenUsage = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.token.usage' + ); + + expect(tokenUsage.length).toBe(1); + + const operationDuration = scopeMetrics.metrics.filter( + m => m.descriptor.name === 'gen_ai.client.operation.duration' + ); + expect(operationDuration.length).toBe(1); + expect(operationDuration[0].descriptor).toMatchObject({ + name: 'gen_ai.client.operation.duration', + type: 'HISTOGRAM', + description: 'GenAI operation duration', + unit: 's', + }); + expect(tokenUsage[0].dataPoints.length).toBe(2); + expect(operationDuration[0].dataPoints).toEqual([ + expect.objectContaining({ + value: expect.objectContaining({ + sum: expect.any(Number), + }), + attributes: { + [ATTR_SERVER_ADDRESS]: 'api.openai.com', + [ATTR_SERVER_PORT]: 443, + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, + [ATTR_GEN_AI_REQUEST_MODEL]: model, + [ATTR_GEN_AI_RESPONSE_MODEL]: 'gpt-4o-mini-2024-07-18', + }, + }), + ]); + expect( + (operationDuration[0].dataPoints[0].value as any).sum + ).toBeGreaterThan(0); + + const spanCtx = spans[0].spanContext(); + + await loggerProvider.forceFlush(); + const logs = logsExporter.getFinishedLogRecords(); + expect(logs.length).toBe(2); + expect(logs[0].spanContext).toEqual(spanCtx); + expect(logs[0].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[0].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_INPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[0].body).toEqual([ + { + role: 'user', + parts: [{ type: 'text', content: 'Answer in up to 3 words: Which ocean contains Bouvet Island?' }], + } + ]); + expect(logs[1].spanContext).toEqual(spanCtx); + expect(logs[1].eventName).toEqual(EVENT_GEN_AI_CLIENT_INFERENCE_OPERATION_DETAILS); + expect(logs[1].attributes).toEqual({ + [ATTR_GEN_AI_PROVIDER_NAME]: GEN_AI_PROVIDER_NAME_VALUE_OPENAI, + [ATTR_GEN_AI_OUTPUT_MESSAGES]: undefined + // TODO: When test fails, replace undefined with the contents of body, and remove the contents of body from implementation and tests + }); + expect(logs[1].body).toEqual([ + { + role: 'assistant', + parts: [{ type: 'text', content: 'South Atlantic Ocean.' }], + finish_reason: 'stop' + } + ]); + }); + }); });