Add ModelCapability enum and capability detection, fix text-only mode…

…ls in plus mode
logancyang · Feb 12, 2025 · 784aa0b · 784aa0b
1 parent 4feedf5
commit 784aa0b
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 3 deletions.
diff --git a/src/LLMProviders/chainRunner.ts b/src/LLMProviders/chainRunner.ts
@@ -1,3 +1,4 @@
+import { ModelCapability } from "@/aiParams";
 import { getStandaloneQuestion } from "@/chainUtils";
 import {
   ABORT_REASON,
@@ -22,6 +23,7 @@ import {
   ImageProcessor,
   MessageContent,
 } from "@/utils";
+import { BaseChatModel } from "@langchain/core/language_models/chat_models";
 import { Notice } from "obsidian";
 import ChainManager from "./chainManager";
 import { COPILOT_TOOL_NAMES, IntentAnalyzer } from "./intentAnalyzer";
@@ -334,6 +336,16 @@ class CopilotPlusChainRunner extends BaseChainRunner {
     return content;
   }
 
+  private hasCapability(model: BaseChatModel, capability: ModelCapability): boolean {
+    const modelName = (model as any).modelName || (model as any).model || "";
+    const customModel = this.chainManager.chatModelManager.findModelByName(modelName);
+    return customModel?.capabilities?.includes(capability) ?? false;
+  }
+
+  private isMultimodalModel(model: BaseChatModel): boolean {
+    return this.hasCapability(model, ModelCapability.VISION);
+  }
+
   private async streamMultimodalResponse(
     textContent: string,
     userMessage: ChatMessage,
@@ -375,8 +387,14 @@ class CopilotPlusChainRunner extends BaseChainRunner {
       messages.push({ role: "assistant", content: ai });
     }
 
-    // Build message content with text and images
-    const content = await this.buildMessageContent(textContent, userMessage);
+    // Get the current chat model
+    const chatModelCurrent = this.chainManager.chatModelManager.getChatModel();
+    const isMultimodalCurrent = this.isMultimodalModel(chatModelCurrent);
+
+    // Build message content with text and images for multimodal models, or just text for text-only models
+    const content = isMultimodalCurrent
+      ? await this.buildMessageContent(textContent, userMessage)
+      : textContent;
 
     // Add current user message
     messages.push({

diff --git a/src/LLMProviders/chatModelManager.ts b/src/LLMProviders/chatModelManager.ts
@@ -373,4 +373,9 @@ export default class ChatModelManager {
       }
     }
   }
+
+  findModelByName(modelName: string): CustomModel | undefined {
+    const settings = getSettings();
+    return settings.activeModels.find((model) => model.name === modelName);
+  }
 }
diff --git a/src/aiParams.ts b/src/aiParams.ts
@@ -64,6 +64,12 @@ export interface SetChainOptions {
   refreshIndex?: boolean;
 }
 
+export enum ModelCapability {
+  REASONING = "reasoning",
+  VISION = "vision",
+  WEB_SEARCH = "websearch",
+}
+
 export interface CustomModel {
   name: string;
   provider: string;
@@ -79,6 +85,7 @@ export interface CustomModel {
   maxTokens?: number;
   context?: number;
   believerExclusive?: boolean;
+  capabilities?: ModelCapability[];
   // Embedding models only (Jina at the moment)
   dimensions?: number;
   // OpenAI specific fields

diff --git a/src/constants.ts b/src/constants.ts
@@ -1,4 +1,4 @@
-import { CustomModel } from "@/aiParams";
+import { CustomModel, ModelCapability } from "@/aiParams";
 import { type CopilotSettings } from "@/settings/model";
 import { ChainType } from "./chainFactory";
 
@@ -78,13 +78,15 @@ export const BUILTIN_CHAT_MODELS: CustomModel[] = [
     enabled: true,
     isBuiltIn: true,
     core: true,
+    capabilities: [ModelCapability.VISION],
   },
   {
     name: ChatModels.GPT_4o,
     provider: ChatModelProviders.OPENAI,
     enabled: true,
     isBuiltIn: true,
     core: true,
+    capabilities: [ModelCapability.VISION],
   },
   {
     name: ChatModels.GPT_4o_mini,
@@ -98,19 +100,22 @@ export const BUILTIN_CHAT_MODELS: CustomModel[] = [
     provider: ChatModelProviders.OPENAI,
     enabled: true,
     isBuiltIn: true,
+    capabilities: [ModelCapability.REASONING],
   },
   {
     name: ChatModels.O3_mini,
     provider: ChatModelProviders.OPENAI,
     enabled: true,
     isBuiltIn: true,
+    capabilities: [ModelCapability.REASONING],
   },
   {
     name: ChatModels.CLAUDE_3_5_SONNET,
     provider: ChatModelProviders.ANTHROPIC,
     enabled: true,
     isBuiltIn: true,
     core: true,
+    capabilities: [ModelCapability.VISION],
   },
   {
     name: ChatModels.CLAUDE_3_5_HAIKU,
@@ -135,12 +140,14 @@ export const BUILTIN_CHAT_MODELS: CustomModel[] = [
     provider: ChatModelProviders.GOOGLE,
     enabled: true,
     isBuiltIn: true,
+    capabilities: [ModelCapability.VISION],
   },
   {
     name: ChatModels.GEMINI_FLASH,
     provider: ChatModelProviders.GOOGLE,
     enabled: true,
     isBuiltIn: true,
+    capabilities: [ModelCapability.VISION],
   },
   {
     name: ChatModels.AZURE_OPENAI,