diff --git a/src/LLMProviders/chainRunner.ts b/src/LLMProviders/chainRunner.ts index f8bf0c7d..c8a683d1 100644 --- a/src/LLMProviders/chainRunner.ts +++ b/src/LLMProviders/chainRunner.ts @@ -1,3 +1,4 @@ +import { ModelCapability } from "@/aiParams"; import { getStandaloneQuestion } from "@/chainUtils"; import { ABORT_REASON, @@ -22,6 +23,7 @@ import { ImageProcessor, MessageContent, } from "@/utils"; +import { BaseChatModel } from "@langchain/core/language_models/chat_models"; import { Notice } from "obsidian"; import ChainManager from "./chainManager"; import { COPILOT_TOOL_NAMES, IntentAnalyzer } from "./intentAnalyzer"; @@ -334,6 +336,16 @@ class CopilotPlusChainRunner extends BaseChainRunner { return content; } + private hasCapability(model: BaseChatModel, capability: ModelCapability): boolean { + const modelName = (model as any).modelName || (model as any).model || ""; + const customModel = this.chainManager.chatModelManager.findModelByName(modelName); + return customModel?.capabilities?.includes(capability) ?? false; + } + + private isMultimodalModel(model: BaseChatModel): boolean { + return this.hasCapability(model, ModelCapability.VISION); + } + private async streamMultimodalResponse( textContent: string, userMessage: ChatMessage, @@ -375,8 +387,14 @@ class CopilotPlusChainRunner extends BaseChainRunner { messages.push({ role: "assistant", content: ai }); } - // Build message content with text and images - const content = await this.buildMessageContent(textContent, userMessage); + // Get the current chat model + const chatModelCurrent = this.chainManager.chatModelManager.getChatModel(); + const isMultimodalCurrent = this.isMultimodalModel(chatModelCurrent); + + // Build message content with text and images for multimodal models, or just text for text-only models + const content = isMultimodalCurrent + ? await this.buildMessageContent(textContent, userMessage) + : textContent; // Add current user message messages.push({ diff --git a/src/LLMProviders/chatModelManager.ts b/src/LLMProviders/chatModelManager.ts index 4d08e3f4..91a5343b 100644 --- a/src/LLMProviders/chatModelManager.ts +++ b/src/LLMProviders/chatModelManager.ts @@ -373,4 +373,9 @@ export default class ChatModelManager { } } } + + findModelByName(modelName: string): CustomModel | undefined { + const settings = getSettings(); + return settings.activeModels.find((model) => model.name === modelName); + } } diff --git a/src/aiParams.ts b/src/aiParams.ts index 16d0b79e..a79f6849 100644 --- a/src/aiParams.ts +++ b/src/aiParams.ts @@ -64,6 +64,12 @@ export interface SetChainOptions { refreshIndex?: boolean; } +export enum ModelCapability { + REASONING = "reasoning", + VISION = "vision", + WEB_SEARCH = "websearch", +} + export interface CustomModel { name: string; provider: string; @@ -79,6 +85,7 @@ export interface CustomModel { maxTokens?: number; context?: number; believerExclusive?: boolean; + capabilities?: ModelCapability[]; // Embedding models only (Jina at the moment) dimensions?: number; // OpenAI specific fields diff --git a/src/constants.ts b/src/constants.ts index 2b5f3c2e..3515005d 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,4 +1,4 @@ -import { CustomModel } from "@/aiParams"; +import { CustomModel, ModelCapability } from "@/aiParams"; import { type CopilotSettings } from "@/settings/model"; import { ChainType } from "./chainFactory"; @@ -78,6 +78,7 @@ export const BUILTIN_CHAT_MODELS: CustomModel[] = [ enabled: true, isBuiltIn: true, core: true, + capabilities: [ModelCapability.VISION], }, { name: ChatModels.GPT_4o, @@ -85,6 +86,7 @@ export const BUILTIN_CHAT_MODELS: CustomModel[] = [ enabled: true, isBuiltIn: true, core: true, + capabilities: [ModelCapability.VISION], }, { name: ChatModels.GPT_4o_mini, @@ -98,12 +100,14 @@ export const BUILTIN_CHAT_MODELS: CustomModel[] = [ provider: ChatModelProviders.OPENAI, enabled: true, isBuiltIn: true, + capabilities: [ModelCapability.REASONING], }, { name: ChatModels.O3_mini, provider: ChatModelProviders.OPENAI, enabled: true, isBuiltIn: true, + capabilities: [ModelCapability.REASONING], }, { name: ChatModels.CLAUDE_3_5_SONNET, @@ -111,6 +115,7 @@ export const BUILTIN_CHAT_MODELS: CustomModel[] = [ enabled: true, isBuiltIn: true, core: true, + capabilities: [ModelCapability.VISION], }, { name: ChatModels.CLAUDE_3_5_HAIKU, @@ -135,12 +140,14 @@ export const BUILTIN_CHAT_MODELS: CustomModel[] = [ provider: ChatModelProviders.GOOGLE, enabled: true, isBuiltIn: true, + capabilities: [ModelCapability.VISION], }, { name: ChatModels.GEMINI_FLASH, provider: ChatModelProviders.GOOGLE, enabled: true, isBuiltIn: true, + capabilities: [ModelCapability.VISION], }, { name: ChatModels.AZURE_OPENAI,