📦 NEW: Vision support

packages/baseai/src/dev/routes/beta/pipes/run.ts

@@ -4,7 +4,7 @@ import { dlog } from '@/dev/utils/dlog';
 import { handleStreamingResponse } from '@/dev/utils/provider-handlers/streaming-response-handler';
 import { logger } from '@/utils/logger-utils';
 import { Hono } from 'hono';
-import { schemaMessage, VariablesSchema } from 'types/pipe';
+import { schemaMessage, schemaPipeMessage, VariablesSchema } from 'types/pipe';
 import { z } from 'zod';
 
 // Schema definitions
@@ -31,7 +31,7 @@ const PipeSchema = z.object({
 	status: z.string(),
 	meta: MetaSchema,
 	model: ModelSchema,
-	messages: z.array(schemaMessage),
+	messages: z.array(schemaPipeMessage),
 	functions: z.array(z.unknown()).default([]),
 	memorysets: z.array(z.string().trim().min(1)).default([]),
 	variables: VariablesSchema

packages/baseai/src/dev/utils/thread/get-message-content.ts

@@ -0,0 +1,36 @@
+import type { Message } from 'types/pipe';
+
+export function getMessageContent(message: Message): string | null {
+	// Tool calls have no content
+	if (!message?.content) return null;
+
+	// If content is a string, return it
+	if (typeof message.content === 'string') {
+		return message.content;
+	}
+
+	/**
+	 * If content is an array, find the text content part and return its text
+	 *
+	 * 1. Image messages have text and image content objects
+	 * {"type": "text", "text": "What’s in this image?"},
+	 *     {
+	 *       "type": "image_url",
+	 *       "image_url": {
+	 *         "url": "",
+	 *       },
+	 *     },
+	 *
+	 * 2. Audio messages always have text and audio content objects
+	 *  content: [
+	 * 		{type: 'text', text: 'What is in this recording?'},
+	 * 		{type: 'input_audio', input_audio: {data: base64str, format: 'wav'}},
+	 * 	];
+	 */
+
+	if (Array.isArray(message.content)) {
+		return message.content.find(item => item.type === 'text')?.text || null;
+	}
+
+	return null;
+}

packages/baseai/src/dev/utils/thread/process-messages.ts

@@ -78,21 +78,41 @@ function replaceVarsInMessagesWithVals({
 		// 1- message.content is empty
 		// 2- message.role is 'assistant'
 		// 3- message.tool_calls is an array of tool calls requested by LLM.
+
+		// 1. If tool call or no content, return msg as is
 		const isAssistantToolCall =
 			!message.content &&
 			message.role === 'assistant' &&
 			message.tool_calls?.length;
+		if (isAssistantToolCall || !message.content) return message;
+
+		// 2. If the content is an array, replace variables in each item
+		if (Array.isArray(message.content)) {
+			const updatedContent = message.content.map(contentItem => ({
+				...contentItem,
 
-		// Since no content to replace variables in, return the message as is.
-		if (isAssistantToolCall) return message;
-		if (!message.content) return message;
+				text: contentItem.text?.replace(
+					variableRegex,
+					(match, varName) => {
+						const trimmedVarName = varName.trim(); // Trim any extra spaces
 
-		// Replace variables in the message content
+						// If the variable exists in the map, replace with its value; otherwise, leave the placeholder intact
+						return variablesMap.get(trimmedVarName) || match;
+					}
+				)
+			}));
+			return {
+				...message,
+				content: updatedContent
+			};
+		}
+
+		// 3. If content is a string, replace variables in it
 		const updatedContent = message.content.replace(
 			variableRegex,
 			(match, varName) => {
-				const trimmedVarName = varName.trim(); // Trim any extra spaces
-				// If the variable exists in the map, replace with its value; otherwise, leave the placeholder intact
+				const trimmedVarName = varName.trim();
+
 				return variablesMap.get(trimmedVarName) || match;
 			}
 		);

packages/baseai/src/utils/memory/lib.ts

@@ -20,6 +20,7 @@ import { loadConfig } from '../config/config-handler';
 import { logger } from '../logger-utils';
 import { generateLocalEmbeddings } from './generate-local-embeddings';
 import { getOpenAIEmbeddings } from './generate-openai-embeddings';
+import { getMessageContent } from '@/dev/utils/thread/get-message-content';
 
 export async function checkDirectoryExists(directoryPath: string) {
 	try {
@@ -155,7 +156,7 @@ export const addContextFromMemory = async ({
 		const lastUserMsg = [...messages]
 			.reverse()
 			.find(m => m.role === 'user');
-		const userPrompt = lastUserMsg?.content;
+		const userPrompt = getMessageContent(lastUserMsg!);
 
 		// If there is no user prompt, return the messages.
 		if (!userPrompt) return;

packages/baseai/types/pipe.ts

@@ -13,7 +13,52 @@ import type {
 } from './model';
 import type { PipeTool } from './tools';
 
+const contentTypeSchema = z.object({
+	type: z.string(),
+	text: z.string().optional(),
+	image_url: z
+		.object({
+			url: z.string(),
+			detail: z.string().optional()
+		})
+		.optional()
+});
+
 export const schemaMessage = z
+	.object({
+		role: z.enum(['system', 'user', 'assistant', 'function', 'tool']),
+		content: z
+			.union([z.string(), z.array(contentTypeSchema), z.null()])
+			.optional(),
+		tool_call_id: z.string().optional(),
+		name: z.string().optional(),
+		tool_calls: z
+			.array(
+				z.object({
+					id: z.string(),
+					type: z.string(),
+					function: z.record(z.unknown())
+				})
+			)
+			.optional()
+	})
+	.refine(
+		({ content, role, tool_calls }) => {
+			// If content is null, role isn't assistant and tool_calls is not present.
+			// then the schema is invalid
+			// because the message content is null and its not an assistant tool call
+			const isSchemaInvalid =
+				content === null && role !== 'assistant' && !tool_calls;
+
+			if (isSchemaInvalid) return false;
+			return true;
+		},
+		{
+			message: 'Message content cannot be empty.'
+		}
+	);
+
+export const schemaPipeMessage = z
 	.object({
 		role: z.enum(['system', 'user', 'assistant', 'function', 'tool']),
 		content: z.string().nullable(),
@@ -46,6 +91,7 @@ export const schemaMessage = z
 	);
 
 export type Message = z.infer<typeof schemaMessage>;
+export type PipeMessage = z.infer<typeof schemaPipeMessage>;
 
 export const VariableSchema = z.object({
 	name: z.string(),
@@ -98,7 +144,7 @@ export interface Pipe {
 	stop: string[];
 	tool_choice: ToolChoice;
 	parallel_tool_calls: boolean;
-	messages: Message[];
+	messages: PipeMessage[];
 	variables: VariablesI;
 	tools: PipeTool[];
 	memory: {