Skip to content

Commit b681d7d

Browse files
feat(ai, openai): expose usage tokens for 'generateImage' function (#10128)
## Background `experimental_generateImage` doesn't expose the token usage information returned by providers. this PR introduces token usage for OpenAI provider We also update the ImageProvider spec so as to allow for usage tokens See #8358 ## Summary - create new type `ImageModelUsage` - update openai image api response schema - map responses to appropriate vars [`inputTokens`, `outputTokens`, `totalTokens`] ## Manual Verification - updated unit tests - updated example in `examples/ai-core/generate-image/openai.ts` - [ ] test with e2e UI example ## Checklist - [x] Tests have been added / updated (for bug fixes / features) - [ ] Documentation has been added / updated (for bug fixes / features) - [ ] A _patch_ changeset for relevant packages has been added (for bug fixes / features - run `pnpm changeset` in the project root) ## Future Work Will need support for other providers for which we support image generation #10150 ## Related Issues Fixes #8358
1 parent 4568ef7 commit b681d7d

File tree

13 files changed

+217
-1
lines changed

13 files changed

+217
-1
lines changed

.changeset/plenty-forks-double.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
'@ai-sdk/provider': patch
3+
'@ai-sdk/openai': patch
4+
'ai': patch
5+
---
6+
7+
feat: expose usage tokens for 'generateImage' function

examples/ai-core/src/generate-image/openai.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ async function main() {
1616
console.log({
1717
prompt,
1818
revisedPrompt,
19+
usage: result.usage,
1920
});
2021

2122
await presentImages([result.image]);

packages/ai/src/generate-image/generate-image-result.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import {
44
ImageModelProviderMetadata,
55
} from '../types/image-model';
66
import { ImageModelResponseMetadata } from '../types/image-model-response-metadata';
7+
import { ImageModelUsage } from '../types/usage';
78

89
/**
910
The result of a `generateImage` call.
@@ -35,4 +36,9 @@ Response metadata from the provider. There may be multiple responses if we made
3536
* results that can be fully encapsulated in the provider.
3637
*/
3738
readonly providerMetadata: ImageModelProviderMetadata;
39+
40+
/**
41+
Combined token usage across all underlying provider calls for this image generation.
42+
*/
43+
readonly usage: ImageModelUsage;
3844
}

packages/ai/src/generate-image/generate-image.test.ts

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -651,4 +651,86 @@ describe('generateImage', () => {
651651
},
652652
});
653653
});
654+
655+
it('should expose empty usage when provider does not report usage', async () => {
656+
const result = await generateImage({
657+
model: new MockImageModelV3({
658+
doGenerate: async () =>
659+
createMockResponse({
660+
images: [pngBase64],
661+
}),
662+
}),
663+
prompt,
664+
});
665+
666+
expect(result.usage).toStrictEqual({
667+
inputTokens: undefined,
668+
outputTokens: undefined,
669+
totalTokens: undefined,
670+
});
671+
});
672+
673+
it('should aggregate usage across multiple provider calls', async () => {
674+
let callCount = 0;
675+
676+
const result = await generateImage({
677+
model: new MockImageModelV3({
678+
maxImagesPerCall: 1,
679+
doGenerate: async () => {
680+
switch (callCount++) {
681+
case 0:
682+
return {
683+
images: [pngBase64],
684+
warnings: [],
685+
providerMetadata: {
686+
testProvider: { images: [null] },
687+
},
688+
response: {
689+
timestamp: new Date(),
690+
modelId: 'mock-model-id',
691+
headers: {},
692+
},
693+
usage: {
694+
inputTokens: 10,
695+
outputTokens: 0,
696+
totalTokens: 10,
697+
},
698+
};
699+
case 1:
700+
return {
701+
images: [jpegBase64],
702+
warnings: [],
703+
providerMetadata: {
704+
testProvider: { images: [null] },
705+
},
706+
response: {
707+
timestamp: new Date(),
708+
modelId: 'mock-model-id',
709+
headers: {},
710+
},
711+
usage: {
712+
inputTokens: 5,
713+
outputTokens: 0,
714+
totalTokens: 5,
715+
},
716+
};
717+
default:
718+
throw new Error('Unexpected call');
719+
}
720+
},
721+
}),
722+
prompt,
723+
n: 2,
724+
});
725+
726+
expect(result.images.map(image => image.base64)).toStrictEqual([
727+
pngBase64,
728+
jpegBase64,
729+
]);
730+
expect(result.usage).toStrictEqual({
731+
inputTokens: 15,
732+
outputTokens: 0,
733+
totalTokens: 15,
734+
});
735+
});
654736
});

packages/ai/src/generate-image/generate-image.ts

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import { ImageModelResponseMetadata } from '../types/image-model-response-metada
1616
import { GenerateImageResult } from './generate-image-result';
1717
import { logWarnings } from '../logger/log-warnings';
1818
import { VERSION } from '../version';
19+
import { addImageModelUsage, ImageModelUsage } from '../types/usage';
1920

2021
/**
2122
Generates images using an image model.
@@ -172,6 +173,11 @@ Only applicable for HTTP-based providers.
172173
const warnings: Array<ImageGenerationWarning> = [];
173174
const responses: Array<ImageModelResponseMetadata> = [];
174175
const providerMetadata: ImageModelV3ProviderMetadata = {};
176+
let totalUsage: ImageModelUsage = {
177+
inputTokens: undefined,
178+
outputTokens: undefined,
179+
totalTokens: undefined,
180+
};
175181
for (const result of results) {
176182
images.push(
177183
...result.images.map(
@@ -188,6 +194,10 @@ Only applicable for HTTP-based providers.
188194
);
189195
warnings.push(...result.warnings);
190196

197+
if (result.usage != null) {
198+
totalUsage = addImageModelUsage(totalUsage, result.usage);
199+
}
200+
191201
if (result.providerMetadata) {
192202
for (const [providerName, metadata] of Object.entries<{
193203
images: unknown;
@@ -213,6 +223,7 @@ Only applicable for HTTP-based providers.
213223
warnings,
214224
responses,
215225
providerMetadata,
226+
usage: totalUsage,
216227
});
217228
}
218229

@@ -221,17 +232,20 @@ class DefaultGenerateImageResult implements GenerateImageResult {
221232
readonly warnings: Array<ImageGenerationWarning>;
222233
readonly responses: Array<ImageModelResponseMetadata>;
223234
readonly providerMetadata: ImageModelV3ProviderMetadata;
235+
readonly usage: ImageModelUsage;
224236

225237
constructor(options: {
226238
images: Array<GeneratedFile>;
227239
warnings: Array<ImageGenerationWarning>;
228240
responses: Array<ImageModelResponseMetadata>;
229241
providerMetadata: ImageModelV3ProviderMetadata;
242+
usage: ImageModelUsage;
230243
}) {
231244
this.images = options.images;
232245
this.warnings = options.warnings;
233246
this.responses = options.responses;
234247
this.providerMetadata = options.providerMetadata;
248+
this.usage = options.usage;
235249
}
236250

237251
get image() {

packages/ai/src/types/index.ts

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,8 @@ export type {
2727
TranscriptionWarning,
2828
} from './transcription-model';
2929
export type { TranscriptionModelResponseMetadata } from './transcription-model-response-metadata';
30-
export type { EmbeddingModelUsage, LanguageModelUsage } from './usage';
30+
export type {
31+
EmbeddingModelUsage,
32+
LanguageModelUsage,
33+
ImageModelUsage,
34+
} from './usage';

packages/ai/src/types/usage.ts

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { LanguageModelV3Usage } from '@ai-sdk/provider';
2+
import { ImageModelV3Usage } from '@ai-sdk/provider';
23

34
/**
45
Represents the number of tokens used in a prompt and completion.
@@ -43,3 +44,19 @@ function addTokenCounts(
4344
? undefined
4445
: (tokenCount1 ?? 0) + (tokenCount2 ?? 0);
4546
}
47+
48+
/**
49+
Usage information for an image model call.
50+
*/
51+
export type ImageModelUsage = ImageModelV3Usage;
52+
53+
export function addImageModelUsage(
54+
usage1: ImageModelUsage,
55+
usage2: ImageModelUsage,
56+
): ImageModelUsage {
57+
return {
58+
inputTokens: addTokenCounts(usage1.inputTokens, usage2.inputTokens),
59+
outputTokens: addTokenCounts(usage1.outputTokens, usage2.outputTokens),
60+
totalTokens: addTokenCounts(usage1.totalTokens, usage2.totalTokens),
61+
};
62+
}

packages/openai/src/image/openai-image-api.ts

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,19 @@ export const openaiImageResponseSchema = lazySchema(() =>
1212
revised_prompt: z.string().nullish(),
1313
}),
1414
),
15+
usage: z
16+
.object({
17+
input_tokens: z.number().nullish(),
18+
output_tokens: z.number().nullish(),
19+
total_tokens: z.number().nullish(),
20+
input_tokens_details: z
21+
.object({
22+
image_tokens: z.number().nullish(),
23+
text_tokens: z.number().nullish(),
24+
})
25+
.nullish(),
26+
})
27+
.nullish(),
1528
}),
1629
),
1730
);

packages/openai/src/image/openai-image-model.test.ts

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,4 +311,42 @@ describe('doGenerate', () => {
311311
},
312312
});
313313
});
314+
315+
it('should map OpenAI usage to usage', async () => {
316+
server.urls['https://api.openai.com/v1/images/generations'].response = {
317+
type: 'json-value',
318+
body: {
319+
created: 1733837122,
320+
data: [
321+
{
322+
b64_json: 'base64-image-1',
323+
},
324+
],
325+
usage: {
326+
input_tokens: 12,
327+
output_tokens: 0,
328+
total_tokens: 12,
329+
input_tokens_details: {
330+
image_tokens: 7,
331+
text_tokens: 5,
332+
},
333+
},
334+
},
335+
};
336+
337+
const result = await provider.image('gpt-image-1').doGenerate({
338+
prompt,
339+
n: 1,
340+
size: '1024x1024',
341+
aspectRatio: undefined,
342+
seed: undefined,
343+
providerOptions: {},
344+
});
345+
346+
expect(result.usage).toStrictEqual({
347+
inputTokens: 12,
348+
outputTokens: 0,
349+
totalTokens: 12,
350+
});
351+
});
314352
});

packages/openai/src/image/openai-image-model.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,14 @@ export class OpenAIImageModel implements ImageModelV3 {
9090
return {
9191
images: response.data.map(item => item.b64_json),
9292
warnings,
93+
usage:
94+
response.usage != null
95+
? {
96+
inputTokens: response.usage.input_tokens ?? undefined,
97+
outputTokens: response.usage.output_tokens ?? undefined,
98+
totalTokens: response.usage.total_tokens ?? undefined,
99+
}
100+
: undefined,
93101
response: {
94102
timestamp: currentDate,
95103
modelId: this.modelId,

0 commit comments

Comments
 (0)