feat(mcp): Implement MCP Gateway integration for extensibility

Implements a simplified MCP (Model Context Protocol) integration approach
as recommended by expert panel (Martin Fowler, Sam Newman, Michael Nygard,
Gregor Hohpe). This provides foundational capability for tool-based
search result enrichment.

Key components:
- ResilientMCPGatewayClient: Thin wrapper (~200 lines) with circuit breaker
  pattern, health checks (5s timeout), retry logic, and graceful degradation
- SearchResultEnricher: Content Enricher pattern implementation (~200 lines)
  with parallel execution and error isolation
- MCP Router: API endpoints for tool discovery and invocation

Features:
- Circuit breaker: 5 failure threshold, 60s recovery timeout
- Health monitoring with 5-second timeout
- API versioning (v1 format)
- Prometheus-ready metrics
- Graceful degradation (core RAG works if tools fail)

Docker infrastructure:
- Redis service for MCP gateway caching
- MCP Context Forge gateway container

Configuration settings added:
- MCP_ENABLED, MCP_GATEWAY_URL, MCP_TIMEOUT
- MCP_CIRCUIT_BREAKER_THRESHOLD, MCP_CIRCUIT_BREAKER_TIMEOUT
- MCP_ENRICHMENT_ENABLED, MCP_MAX_CONCURRENT

Closes #653

Author: claude

backend/core/config.py

@@ -284,6 +284,28 @@ class Settings(BaseSettings):
     log_storage_enabled: Annotated[bool, Field(default=True, alias="LOG_STORAGE_ENABLED")]
     log_buffer_size_mb: Annotated[int, Field(default=5, alias="LOG_BUFFER_SIZE_MB")]
 
+    # MCP Gateway settings
+    # Enable/disable MCP integration globally
+    mcp_enabled: Annotated[bool, Field(default=True, alias="MCP_ENABLED")]
+    # MCP Context Forge gateway URL
+    mcp_gateway_url: Annotated[str, Field(default="http://localhost:3000", alias="MCP_GATEWAY_URL")]
+    # Request timeout in seconds (30s default per requirements)
+    mcp_timeout: Annotated[float, Field(default=30.0, ge=1.0, le=300.0, alias="MCP_TIMEOUT")]
+    # Health check timeout (5s per requirements)
+    mcp_health_timeout: Annotated[float, Field(default=5.0, ge=1.0, le=30.0, alias="MCP_HEALTH_TIMEOUT")]
+    # Maximum retries for MCP calls
+    mcp_max_retries: Annotated[int, Field(default=3, ge=0, le=10, alias="MCP_MAX_RETRIES")]
+    # Circuit breaker failure threshold (5 failures per requirements)
+    mcp_circuit_breaker_threshold: Annotated[int, Field(default=5, ge=1, le=20, alias="MCP_CIRCUIT_BREAKER_THRESHOLD")]
+    # Circuit breaker recovery timeout in seconds (60s per requirements)
+    mcp_circuit_breaker_timeout: Annotated[float, Field(default=60.0, ge=10.0, le=600.0, alias="MCP_CIRCUIT_BREAKER_TIMEOUT")]
+    # JWT token for MCP gateway authentication
+    mcp_jwt_token: Annotated[str | None, Field(default=None, alias="MCP_JWT_TOKEN")]
+    # Enable enrichment of search results with MCP tools
+    mcp_enrichment_enabled: Annotated[bool, Field(default=True, alias="MCP_ENRICHMENT_ENABLED")]
+    # Maximum concurrent MCP tool invocations
+    mcp_max_concurrent: Annotated[int, Field(default=5, ge=1, le=20, alias="MCP_MAX_CONCURRENT")]
+
     # Testing settings
     testing: Annotated[bool, Field(default=False, alias="TESTING")]
     skip_auth: Annotated[bool, Field(default=False, alias="SKIP_AUTH")]

backend/main.py

@@ -34,6 +34,7 @@
 
 # Routers
 from rag_solution.router.chat_router import router as chat_router
+from rag_solution.router.mcp_router import router as mcp_router
 from rag_solution.router.collection_router import router as collection_router
 from rag_solution.router.conversation_router import router as conversation_router
 from rag_solution.router.dashboard_router import router as dashboard_router
@@ -248,6 +249,7 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:
 app.include_router(auth_router)
 app.include_router(chat_router)
 app.include_router(conversation_router)
+app.include_router(mcp_router)
 app.include_router(dashboard_router)
 app.include_router(health_router)
 app.include_router(collection_router)

backend/rag_solution/router/mcp_router.py

@@ -0,0 +1,275 @@
+"""MCP Gateway router for RAG Modulo API.
+
+This module provides FastAPI router endpoints for MCP (Model Context Protocol)
+Gateway integration, enabling tool discovery and invocation capabilities.
+
+API Endpoints:
+- GET /api/v1/mcp/tools - List available MCP tools
+- POST /api/v1/mcp/tools/{name}/invoke - Invoke a specific MCP tool
+- GET /api/v1/mcp/health - Check MCP gateway health
+"""
+
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, HTTPException, status
+
+from core.config import Settings, get_settings
+from core.logging_utils import get_logger
+from rag_solution.core.dependencies import get_current_user
+from rag_solution.schemas.mcp_schema import (
+    MCPHealthStatus,
+    MCPInvocationInput,
+    MCPInvocationOutput,
+    MCPInvocationStatus,
+    MCPToolsResponse,
+)
+from rag_solution.services.mcp_gateway_client import ResilientMCPGatewayClient
+
+logger = get_logger(__name__)
+
+router = APIRouter(prefix="/api/v1/mcp", tags=["mcp"])
+
+
+def get_mcp_client(
+    settings: Annotated[Settings, Depends(get_settings)],
+) -> ResilientMCPGatewayClient:
+    """Dependency to create MCP gateway client.
+
+    Args:
+        settings: Application settings from dependency injection
+
+    Returns:
+        ResilientMCPGatewayClient: Initialized MCP client instance
+    """
+    return ResilientMCPGatewayClient(settings)
+
+
+@router.get(
+    "/health",
+    response_model=MCPHealthStatus,
+    summary="Check MCP gateway health",
+    description="Perform a health check on the MCP Context Forge gateway",
+    responses={
+        200: {"description": "Health check completed (see healthy field for status)"},
+        503: {"description": "MCP integration is disabled"},
+    },
+)
+async def mcp_health(
+    settings: Annotated[Settings, Depends(get_settings)],
+    mcp_client: Annotated[ResilientMCPGatewayClient, Depends(get_mcp_client)],
+) -> MCPHealthStatus:
+    """Check MCP gateway health status.
+
+    Returns health information including:
+    - Gateway availability
+    - Latency
+    - Circuit breaker state
+
+    Args:
+        settings: Application settings
+        mcp_client: MCP gateway client
+
+    Returns:
+        MCPHealthStatus: Health status information
+    """
+    if not settings.mcp_enabled:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail="MCP integration is disabled",
+        )
+
+    return await mcp_client.check_health()
+
+
+@router.get(
+    "/tools",
+    response_model=MCPToolsResponse,
+    summary="List available MCP tools",
+    description="Retrieve a list of all available MCP tools from the gateway",
+    responses={
+        200: {"description": "List of available MCP tools"},
+        503: {"description": "MCP integration is disabled or gateway unavailable"},
+    },
+)
+async def list_tools(
+    current_user: Annotated[dict, Depends(get_current_user)],
+    settings: Annotated[Settings, Depends(get_settings)],
+    mcp_client: Annotated[ResilientMCPGatewayClient, Depends(get_mcp_client)],
+) -> MCPToolsResponse:
+    """List all available MCP tools.
+
+    Returns tools available for invocation, including their:
+    - Name and description
+    - Input parameters
+    - Category and version
+
+    SECURITY: Requires authentication.
+
+    Args:
+        current_user: Authenticated user from JWT token
+        settings: Application settings
+        mcp_client: MCP gateway client
+
+    Returns:
+        MCPToolsResponse: List of available tools
+
+    Raises:
+        HTTPException: If MCP is disabled or gateway unavailable
+    """
+    if not settings.mcp_enabled:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail="MCP integration is disabled",
+        )
+
+    logger.info(
+        "Listing MCP tools",
+        extra={
+            "user_id": current_user.get("uuid"),
+        },
+    )
+
+    response = await mcp_client.list_tools()
+
+    if not response.gateway_healthy:
+        logger.warning(
+            "MCP gateway unhealthy when listing tools",
+            extra={
+                "user_id": current_user.get("uuid"),
+            },
+        )
+
+    return response
+
+
+@router.post(
+    "/tools/{tool_name}/invoke",
+    response_model=MCPInvocationOutput,
+    summary="Invoke an MCP tool",
+    description="Invoke a specific MCP tool with the provided arguments",
+    responses={
+        200: {"description": "Tool invocation completed (check status field)"},
+        400: {"description": "Invalid input data"},
+        404: {"description": "Tool not found"},
+        503: {"description": "MCP integration is disabled"},
+    },
+)
+async def invoke_tool(
+    tool_name: str,
+    invocation_input: MCPInvocationInput,
+    current_user: Annotated[dict, Depends(get_current_user)],
+    settings: Annotated[Settings, Depends(get_settings)],
+    mcp_client: Annotated[ResilientMCPGatewayClient, Depends(get_mcp_client)],
+) -> MCPInvocationOutput:
+    """Invoke a specific MCP tool.
+
+    Executes the named tool with provided arguments. Implements graceful
+    degradation - tool failures are returned in the response status rather
+    than throwing exceptions (except for validation errors).
+
+    SECURITY: Requires authentication.
+
+    Args:
+        tool_name: Name of the tool to invoke
+        invocation_input: Tool arguments and optional timeout
+        current_user: Authenticated user from JWT token
+        settings: Application settings
+        mcp_client: MCP gateway client
+
+    Returns:
+        MCPInvocationOutput: Tool execution result
+
+    Raises:
+        HTTPException: If MCP is disabled or input validation fails
+    """
+    if not settings.mcp_enabled:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail="MCP integration is disabled",
+        )
+
+    if not tool_name or not tool_name.strip():
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Tool name is required",
+        )
+
+    user_id = current_user.get("uuid")
+
+    logger.info(
+        "Invoking MCP tool",
+        extra={
+            "tool_name": tool_name,
+            "user_id": user_id,
+            "has_arguments": bool(invocation_input.arguments),
+        },
+    )
+
+    result = await mcp_client.invoke_tool(
+        tool_name=tool_name.strip(),
+        arguments=invocation_input.arguments,
+        timeout=invocation_input.timeout,
+    )
+
+    # Log result status
+    if result.status == MCPInvocationStatus.SUCCESS:
+        logger.info(
+            "MCP tool invocation succeeded",
+            extra={
+                "tool_name": tool_name,
+                "user_id": user_id,
+                "execution_time_ms": result.execution_time_ms,
+            },
+        )
+    else:
+        logger.warning(
+            "MCP tool invocation failed",
+            extra={
+                "tool_name": tool_name,
+                "user_id": user_id,
+                "status": result.status.value,
+                "error": result.error,
+            },
+        )
+
+    return result
+
+
+@router.get(
+    "/metrics",
+    summary="Get MCP client metrics",
+    description="Retrieve Prometheus-ready metrics from the MCP client",
+    responses={
+        200: {"description": "Client metrics"},
+        503: {"description": "MCP integration is disabled"},
+    },
+)
+async def get_metrics(
+    current_user: Annotated[dict, Depends(get_current_user)],
+    settings: Annotated[Settings, Depends(get_settings)],
+    mcp_client: Annotated[ResilientMCPGatewayClient, Depends(get_mcp_client)],
+) -> dict:
+    """Get MCP client metrics for monitoring.
+
+    Returns Prometheus-ready metrics including:
+    - Request counts (total, success, failed)
+    - Circuit breaker state
+    - Health check statistics
+
+    SECURITY: Requires authentication.
+
+    Args:
+        current_user: Authenticated user from JWT token
+        settings: Application settings
+        mcp_client: MCP gateway client
+
+    Returns:
+        dict: Client metrics
+    """
+    if not settings.mcp_enabled:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail="MCP integration is disabled",
+        )
+
+    return mcp_client.get_metrics()