Fix media file prompting for vertex gemini and google gemini

src/providers/google-vertex-ai/chatComplete.ts

@@ -116,19 +116,27 @@ export const VertexGoogleChatCompleteConfig: ProviderConfig = {
                   });
 
                   return;
+                } else if (
+                  url.startsWith('gs://') ||
+                  url.startsWith('https://') ||
+                  url.startsWith('http://')
+                ) {
+                  parts.push({
+                    fileData: {
+                      mimeType: getMimeType(url),
+                      fileUri: url,
+                    },
+                  });
+                } else {
+                  // NOTE: This block is kept to maintain backward compatibility
+                  // Earlier we were assuming that all images will be base64 with image/jpeg mimeType
+                  parts.push({
+                    inlineData: {
+                      mimeType: 'image/jpeg',
+                      data: c.image_url?.url,
+                    },
+                  });
                 }
-
-                // This part is problematic because URLs are not supported in the current implementation.
-                // Two problems exist:
-                // 1. Only Google Cloud Storage URLs are supported.
-                // 2. MimeType is not supported in OpenAI API, but it is required in Google Vertex AI API.
-                // Google will return an error here if any other URL is provided.
-                parts.push({
-                  fileData: {
-                    mimeType: getMimeType(url),
-                    fileUri: url,
-                  },
-                });
               }
             });
           } else if (typeof message.content === 'string') {

src/providers/google-vertex-ai/utils.ts

@@ -162,12 +162,13 @@ const fileExtensionMimeTypeMap = {
   flv: 'video/flv',
 };
 
-export const getMimeType = (url: string) => {
+export const getMimeType = (url: string): string | undefined => {
   const urlParts = url.split('.');
   const extension = urlParts[
     urlParts.length - 1
   ] as keyof typeof fileExtensionMimeTypeMap;
-  return fileExtensionMimeTypeMap[extension] || 'image/jpeg';
+  const mimeType = fileExtensionMimeTypeMap[extension];
+  return mimeType;
 };
 
 export const GoogleErrorResponseTransform: (

src/providers/google/chatComplete.ts

@@ -188,7 +188,11 @@ export const GoogleChatCompleteConfig: ProviderConfig = {
                       data: base64Image,
                     },
                   });
-                } else if (url.startsWith('gs://')) {
+                } else if (
+                  url.startsWith('gs://') ||
+                  url.startsWith('https://') ||
+                  url.startsWith('http://')
+                ) {
                   parts.push({
                     fileData: {
                       mimeType: getMimeType(url),