feat (provider/openai): send user message content as text when possib…

…le (#1765)

Co-authored-by: minpeter <minpeter@friendli.ai>
Co-authored-by: minpeter <kali2005611@gmail.com>

.changeset/kind-pugs-flow.md

@@ -0,0 +1,5 @@
+---
+'@ai-sdk/openai': patch
+---
+
+feat (provider/openai): send user message content as text when possible

packages/openai/src/convert-to-openai-chat-messages.test.ts

@@ -0,0 +1,43 @@
+import { convertToOpenAIChatMessages } from './convert-to-openai-chat-messages';
+
+describe('user messages', () => {
+  it('should convert messages with image parts to multiple parts', async () => {
+    const result = convertToOpenAIChatMessages([
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: 'Hello' },
+          {
+            type: 'image',
+            image: new Uint8Array([0, 1, 2, 3]),
+            mimeType: 'image/png',
+          },
+        ],
+      },
+    ]);
+
+    expect(result).toEqual([
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: 'Hello' },
+          {
+            type: 'image_url',
+            image_url: { url: 'data:image/png;base64,AAECAw==' },
+          },
+        ],
+      },
+    ]);
+  });
+
+  it('should convert messages with only a text part to a string content', async () => {
+    const result = convertToOpenAIChatMessages([
+      {
+        role: 'user',
+        content: [{ type: 'text', text: 'Hello' }],
+      },
+    ]);
+
+    expect(result).toEqual([{ role: 'user', content: 'Hello' }]);
+  });
+});

packages/openai/src/convert-to-openai-chat-messages.ts

@@ -15,6 +15,11 @@ export function convertToOpenAIChatMessages(
       }
 
       case 'user': {
+        if (content.length === 1 && content[0].type === 'text') {
+          messages.push({ role: 'user', content: content[0].text });
+          break;
+        }
+
         messages.push({
           role: 'user',
           content: content.map(part => {
@@ -38,6 +43,7 @@ export function convertToOpenAIChatMessages(
             }
           }),
         });
+
         break;
       }
 

packages/openai/src/openai-chat-language-model.test.ts

@@ -263,7 +263,7 @@ describe('doGenerate', () => {
 
     expect(await server.getRequestBodyJson()).toStrictEqual({
       model: 'gpt-3.5-turbo',
-      messages: [{ role: 'user', content: [{ type: 'text', text: 'Hello' }] }],
+      messages: [{ role: 'user', content: 'Hello' }],
       logprobs: false,
     });
   });
@@ -298,7 +298,7 @@ describe('doGenerate', () => {
 
     expect(await server.getRequestBodyJson()).toStrictEqual({
       model: 'gpt-3.5-turbo',
-      messages: [{ role: 'user', content: [{ type: 'text', text: 'Hello' }] }],
+      messages: [{ role: 'user', content: 'Hello' }],
       logprobs: false,
       tools: [
         {
@@ -612,7 +612,7 @@ describe('doStream', () => {
       stream: true,
       stream_options: { include_usage: true },
       model: 'gpt-3.5-turbo',
-      messages: [{ role: 'user', content: [{ type: 'text', text: 'Hello' }] }],
+      messages: [{ role: 'user', content: 'Hello' }],
       logprobs: false,
     });
   });

packages/openai/src/openai-chat-language-model.ts

@@ -403,7 +403,6 @@ const openAIChatResponseSchema = z.object({
       finish_reason: z.string().optional().nullable(),
     }),
   ),
-  object: z.literal('chat.completion'),
   usage: z.object({
     prompt_tokens: z.number(),
     completion_tokens: z.number(),