feat (provider/openai): support ai sdk image download (#2914)

.changeset/purple-bulldogs-admire.md

@@ -0,0 +1,5 @@
+---
+'@ai-sdk/openai': patch
+---
+
+feat (provider/openai): support ai sdk image download

content/providers/01-ai-sdk-providers/01-openai.mdx

@@ -186,7 +186,7 @@ The following optional settings are available for OpenAI chat models:
 
 - **parallelToolCalls** _boolean_
 
-  Whether to enable parallel function calling during tool use. Default to true.
+  Whether to enable parallel function calling during tool use. Defaults to `true`.
 
 - **useLegacyFunctionCalls** _boolean_
 
@@ -200,7 +200,7 @@ The following optional settings are available for OpenAI chat models:
 
 - **structuredOutputs** _boolean_
 
-  Whether to use [structured outputs](#structured-outputs). Defaults to false.
+  Whether to use [structured outputs](#structured-outputs). Defaults to `false`.
 
   When enabled, tool calls and object generation will be strict and follow the provided schema.
 
@@ -209,6 +209,13 @@ The following optional settings are available for OpenAI chat models:
   A unique identifier representing your end-user, which can help OpenAI to
   monitor and detect abuse. Learn more.
 
+- **downloadImages** _boolean_
+
+  Automatically download images and pass the image as data to the model.
+  OpenAI supports image URLs for public models, so this is only needed for
+  private models or when the images are not publicly accessible.
+  Defaults to `false`.
+
 #### Structured Outputs
 
 You can enable [OpenAI structured outputs](https://openai.com/index/introducing-structured-outputs-in-the-api/) by setting the `structuredOutputs` option to `true`.

packages/openai/src/openai-chat-language-model.test.ts

@@ -113,6 +113,27 @@ const provider = createOpenAI({
 
 const model = provider.chat('gpt-3.5-turbo');
 
+describe('settings', () => {
+  it('should set supportsImageUrls to true by default', () => {
+    const defaultModel = provider.chat('gpt-3.5-turbo');
+    expect(defaultModel.supportsImageUrls).toBe(true);
+  });
+
+  it('should set supportsImageUrls to false when downloadImages is true', () => {
+    const modelWithDownloadImages = provider.chat('gpt-3.5-turbo', {
+      downloadImages: true,
+    });
+    expect(modelWithDownloadImages.supportsImageUrls).toBe(false);
+  });
+
+  it('should set supportsImageUrls to true when downloadImages is false', () => {
+    const modelWithoutDownloadImages = provider.chat('gpt-3.5-turbo', {
+      downloadImages: false,
+    });
+    expect(modelWithoutDownloadImages.supportsImageUrls).toBe(true);
+  });
+});
+
 describe('doGenerate', () => {
   const server = new JsonTestServer(
     'https://api.openai.com/v1/chat/completions',

packages/openai/src/openai-chat-language-model.ts

@@ -66,6 +66,11 @@ export class OpenAIChatLanguageModel implements LanguageModelV1 {
     return this.config.provider;
   }
 
+  get supportsImageUrls(): boolean {
+    // image urls can be sent if downloadImages is disabled (default):
+    return !this.settings.downloadImages;
+  }
+
   private getArgs({
     mode,
     prompt,

packages/openai/src/openai-chat-settings.ts

@@ -77,4 +77,13 @@ A unique identifier representing your end-user, which can help OpenAI to
 monitor and detect abuse. Learn more.
 */
   user?: string;
+
+  /**
+Automatically download images and pass the image as data to the model.
+OpenAI supports image URLs for public models, so this is only needed for
+private models or when the images are not publicly accessible.
+
+Defaults to `false`.
+   */
+  downloadImages?: boolean;
 }