Add initial support for MCP (#14598)

* Add initial support for MCP

fixed #14523

Signed-off-by: Jonas Helming <jhelming@eclipsesource.com>
Co-authored-by: Stefan Dirix <sdirix@eclipsesource.com>

examples/browser/package.json

@@ -30,6 +30,7 @@
     "@theia/ai-history": "1.56.0",
     "@theia/ai-huggingface": "1.56.0",
     "@theia/ai-llamafile": "1.56.0",
+    "@theia/ai-mcp": "1.56.0",
     "@theia/ai-ollama": "1.56.0",
     "@theia/ai-openai": "1.56.0",
     "@theia/ai-terminal": "1.56.0",

examples/browser/tsconfig.json

@@ -32,6 +32,9 @@
     {
       "path": "../../packages/ai-llamafile"
     },
+    {
+      "path": "../../packages/ai-mcp"
+    },
     {
       "path": "../../packages/ai-ollama"
     },

packages/ai-core/src/browser/prompttemplate-contribution.ts

@@ -14,15 +14,15 @@
 // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
 // *****************************************************************************
 
-import { inject, injectable, named } from '@theia/core/shared/inversify';
+import { inject, injectable } from '@theia/core/shared/inversify';
 import { GrammarDefinition, GrammarDefinitionProvider, LanguageGrammarDefinitionContribution, TextmateRegistry } from '@theia/monaco/lib/browser/textmate';
 import * as monaco from '@theia/monaco-editor-core';
-import { Command, CommandContribution, CommandRegistry, ContributionProvider, MessageService } from '@theia/core';
+import { Command, CommandContribution, CommandRegistry, MessageService } from '@theia/core';
 import { TabBarToolbarContribution, TabBarToolbarRegistry } from '@theia/core/lib/browser/shell/tab-bar-toolbar';
 
 import { codicon, Widget } from '@theia/core/lib/browser';
 import { EditorWidget, ReplaceOperation } from '@theia/editor/lib/browser';
-import { PromptCustomizationService, PromptService, ToolProvider } from '../common';
+import { PromptCustomizationService, PromptService, ToolInvocationRegistry } from '../common';
 import { ProviderResult } from '@theia/monaco-editor-core/esm/vs/editor/common/languages';
 
 const PROMPT_TEMPLATE_LANGUAGE_ID = 'theia-ai-prompt-template';
@@ -56,9 +56,8 @@ export class PromptTemplateContribution implements LanguageGrammarDefinitionCont
     @inject(PromptCustomizationService)
     protected readonly customizationService: PromptCustomizationService;
 
-    @inject(ContributionProvider)
-    @named(ToolProvider)
-    private toolProviders: ContributionProvider<ToolProvider>;
+    @inject(ToolInvocationRegistry)
+    protected readonly toolInvocationRegistry: ToolInvocationRegistry;
 
     readonly config: monaco.languages.LanguageConfiguration =
         {
@@ -115,7 +114,7 @@ export class PromptTemplateContribution implements LanguageGrammarDefinitionCont
             model,
             position,
             '~{',
-            this.toolProviders.getContributions().map(provider => provider.getTool()),
+            this.toolInvocationRegistry.getAllFunctions(),
             monaco.languages.CompletionItemKind.Function,
             tool => tool.id,
             tool => tool.name,

packages/ai-core/src/common/language-model.ts

@@ -32,13 +32,40 @@ export const isLanguageModelRequestMessage = (obj: unknown): obj is LanguageMode
         'query' in obj &&
         typeof (obj as { query: unknown }).query === 'string'
     );
+export type ToolRequestParametersProperties = Record<string, { type: string, [key: string]: unknown }>;
+export interface ToolRequestParameters {
+    type?: 'object';
+    properties: ToolRequestParametersProperties
+}
 export interface ToolRequest {
     id: string;
     name: string;
-    parameters?: { type?: 'object', properties: Record<string, { type: string, [key: string]: unknown }> };
+    parameters?: ToolRequestParameters
     description?: string;
     handler: (arg_string: string) => Promise<unknown>;
+    providerName?: string;
 }
+
+export namespace ToolRequest {
+    export function isToolRequestParametersProperties(obj: unknown): obj is ToolRequestParametersProperties {
+        if (!obj || typeof obj !== 'object') { return false; };
+
+        return Object.entries(obj).every(([key, value]) =>
+            typeof key === 'string' &&
+            value &&
+            typeof value === 'object' &&
+            'type' in value &&
+            typeof value.type === 'string' &&
+            Object.keys(value).every(k => typeof k === 'string')
+        );
+    }
+    export function isToolRequestParameters(obj: unknown): obj is ToolRequestParameters {
+        return !!obj && typeof obj === 'object' &&
+            (!('type' in obj) || obj.type === 'object') &&
+            'properties' in obj && isToolRequestParametersProperties(obj.properties);
+    }
+}
+
 export interface LanguageModelRequest {
     messages: LanguageModelRequestMessage[],
     tools?: ToolRequest[];

packages/ai-core/src/common/tool-invocation-registry.ts

@@ -24,11 +24,44 @@ export const ToolInvocationRegistry = Symbol('ToolInvocationRegistry');
  * Registry for all the function calls available to Agents.
  */
 export interface ToolInvocationRegistry {
+    /**
+     * Registers a tool into the registry.
+     *
+     * @param tool - The `ToolRequest` object representing the tool to be registered.
+     */
     registerTool(tool: ToolRequest): void;
 
+    /**
+     * Retrieves a specific `ToolRequest` from the registry.
+     *
+     * @param toolId - The unique identifier of the tool to retrieve.
+     * @returns The `ToolRequest` object corresponding to the provided tool ID,
+     *          or `undefined` if the tool is not found in the registry.
+     */
     getFunction(toolId: string): ToolRequest | undefined;
 
+    /**
+     * Retrieves multiple `ToolRequest`s from the registry.
+     *
+     * @param toolIds - A list of tool IDs to retrieve.
+     * @returns An array of `ToolRequest` objects for the specified tool IDs.
+     *          If a tool ID is not found, it is skipped in the returned array.
+     */
     getFunctions(...toolIds: string[]): ToolRequest[];
+
+    /**
+     * Retrieves all `ToolRequest`s currently registered in the registry.
+     *
+     * @returns An array of all `ToolRequest` objects in the registry.
+     */
+    getAllFunctions(): ToolRequest[];
+
+    /**
+     * Unregisters all tools provided by a specific tool provider.
+     *
+     * @param providerName - The name of the tool provider whose tools should be removed (as specificed in the `ToolRequest`).
+     */
+    unregisterAllTools(providerName: string): void;
 }
 
 export const ToolProvider = Symbol('ToolProvider');
@@ -52,6 +85,19 @@ export class ToolInvocationRegistryImpl implements ToolInvocationRegistry {
         });
     }
 
+    unregisterAllTools(providerName: string): void {
+        const toolsToRemove: string[] = [];
+        for (const [id, tool] of this.tools.entries()) {
+            if (tool.providerName === providerName) {
+                toolsToRemove.push(id);
+            }
+        }
+        toolsToRemove.forEach(id => this.tools.delete(id));
+    }
+    getAllFunctions(): ToolRequest[] {
+        return Array.from(this.tools.values());
+    }
+
     registerTool(tool: ToolRequest): void {
         if (this.tools.has(tool.id)) {
             console.warn(`Function with id ${tool.id} is already registered.`);

packages/ai-mcp/.eslintrc.js

@@ -0,0 +1,10 @@
+/** @type {import('eslint').Linter.Config} */
+module.exports = {
+    extends: [
+        '../../configs/build.eslintrc.json'
+    ],
+    parserOptions: {
+        tsconfigRootDir: __dirname,
+        project: 'tsconfig.json'
+    }
+};

packages/ai-mcp/README.md

@@ -0,0 +1,91 @@
+# Model Context Server (MCP) Integration
+
+The AI MCP package provides an integration that allows users to start and use MCP servers to provide additional tool functions to LLMs, e.g. search or file access (outside of the workspace).
+
+## Features
+-   Add MCP servers via settings.json
+-   Start and stop MCP servers.
+-   Use tool functions provided by MCP servers in prompt templates
+
+## Commands
+
+### Start MCP Server
+
+-   **Command ID:** `mcp.startserver`
+-   **Label:** `MCP: Start MCP Server`
+-   **Functionality:** Allows you to start a MCP server by selecting from a list of configured servers.
+
+### Stop MCP Server
+
+-   **Command ID:** `mcp.stopserver`
+-   **Label:** `MCP: Stop MCP Server`
+-   **Functionality:** Allows you to stop a running MCP server by selecting from a list of currently running servers.
+
+## Usage
+
+1. **Starting a MCP Server:**
+
+    - Use the command palette to invoke `MCP: Start MCP Server`.
+    - A quick pick menu will appear with a list of configured MCP servers.
+    - Select a server to start.
+
+2. **Stopping a MCP Server:**
+    - Use the command palette to invoke `MCP: Stop MCP Server`.
+    - A quick pick menu will display a list of currently running MCP servers.
+    - Select a server to stop.
+
+3. **Using provided tool functions**
+    - Only functions of started MCP servers can be used
+    - Open a prompt template and add the added tool functions
+    - Type '~{' to open the auto completion 
+
+## Configuration
+
+Make sure to configure your MCP servers properly within the preference settings.
+
+Example Configuration:
+
+```json
+{
+    "ai-features.mcp.mcpServers": {
+        "memory": {
+            "command": "npx",
+            "args": [
+              "-y",
+              "@modelcontextprotocol/server-memory"
+            ]
+          },
+          "brave-search": {
+            "command": "npx",
+            "args": [
+              "-y",
+              "@modelcontextprotocol/server-brave-search"
+            ],
+            "env": {
+              "BRAVE_API_KEY": "YOUR_API_KEY"
+            }
+          },
+          "filesystem": {
+            "command": "npx",
+            "args": [
+              "-y",
+              "@modelcontextprotocol/server-filesystem",
+              "ABSOLUTE_PATH_TO_ALLOWED_DIRECTORY",
+            ]
+          },
+    }
+}
+```
+
+Example prompt (for search)
+```md
+~{mcp_brave-search_brave_web_search}
+```
+
+Example User query
+```md
+Search the internet for XYZ
+```
+
+## More Information
+[List of available MCP servers](https://github.com/modelcontextprotocol/servers)

packages/ai-mcp/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@theia/ai-mcp",
+  "version": "1.56.0",
+  "description": "Theia - MCP Integration",
+  "dependencies": {
+    "@theia/core": "1.56.0",
+    "@theia/ai-core": "1.56.0",
+    "@modelcontextprotocol/sdk": "1.0.1"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "theiaExtensions": [
+    {
+      "frontend": "lib/browser/mcp-frontend-module",
+      "backend": "lib/node/mcp-backend-module"
+    }
+  ],
+  "keywords": [
+    "theia-extension"
+  ],
+  "license": "EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/eclipse-theia/theia.git"
+  },
+  "bugs": {
+    "url": "https://github.com/eclipse-theia/theia/issues"
+  },
+  "homepage": "https://github.com/eclipse-theia/theia",
+  "files": [
+    "lib",
+    "src"
+  ],
+  "scripts": {
+    "build": "theiaext build",
+    "clean": "theiaext clean",
+    "compile": "theiaext compile",
+    "lint": "theiaext lint",
+    "test": "theiaext test",
+    "watch": "theiaext watch"
+  },
+  "devDependencies": {
+    "@theia/ext-scripts": "1.56.0"
+  },
+  "nyc": {
+    "extends": "../../configs/nyc.json"
+  }
+}