diff --git a/CHANGELOG.md b/CHANGELOG.md index ca4461538..863e2c78a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,68 @@ --- +## [0.9.0] - In Progress (Due: 2025-11-04) - REST Passthrough Enhancements & Advanced Tool Management + +### Overview + +This release focuses on **REST API Passthrough Capabilities** and **Advanced Tool Management** with enhanced configurability for REST integrations: + +- **🔌 REST Passthrough API Fields** - Comprehensive REST tool configuration with query/header mapping, timeouts, and plugin chains +- **🎯 Advanced Tool Configuration** - Fine-grained control over REST tool behavior and request customization +- **🔒 Security & Validation** - Enhanced validation for REST endpoints with allowlists and configurable timeouts + +### Added + +#### **🔌 REST Passthrough Configuration** (#746, #1273) +* **Query & Header Mapping** - Configure dynamic query parameter and header mappings for REST tools +* **Path Templates** - Support for URL path templates with variable substitution +* **Timeout Configuration** - Per-tool timeout settings (default: 20000ms for REST passthrough) +* **Endpoint Exposure Control** - Toggle passthrough endpoint visibility with `expose_passthrough` flag +* **Host Allowlists** - Configure allowed upstream hosts/schemes for enhanced security +* **Plugin Chain Support** - Pre and post-request plugin chains for REST tools +* **Base URL Extraction** - Automatic extraction of base URL and path template from tool URLs +* **Admin UI Integration** - "Advanced: Add Passthrough" button in tool creation form with dynamic field generation + +#### **🛡️ REST Tool Validation** (#1273) +* **URL Structure Validation** - Ensures base URLs have valid scheme and netloc +* **Path Template Validation** - Enforces leading slash in path templates +* **Timeout Validation** - Validates timeout values are positive integers +* **Allowlist Validation** - Regex-based validation for allowed hosts/schemes +* **Plugin Chain Validation** - Restricts plugins to known safe plugins (deny_filter, rate_limit, pii_filter, response_shape, regex_filter, resource_filter) +* **Integration Type Enforcement** - REST-specific fields only allowed for `integration_type='REST'` + +### Changed + +#### **📊 Database Schema** (#1273) +* **New Tool Columns** - Added 9 new columns to tools table via Alembic migration `8a2934be50c0`: + - `base_url` - Base URL for REST passthrough + - `path_template` - Path template for URL construction + - `query_mapping` - JSON mapping for query parameters + - `header_mapping` - JSON mapping for headers + - `timeout_ms` - Request timeout in milliseconds + - `expose_passthrough` - Boolean flag to enable/disable passthrough + - `allowlist` - JSON array of allowed hosts/schemes + - `plugin_chain_pre` - Pre-request plugin chain + - `plugin_chain_post` - Post-request plugin chain + +#### **🔧 API Schemas** (#1273) +* **ToolCreate Schema** - Enhanced with passthrough field validation and auto-extraction logic +* **ToolUpdate Schema** - Updated with same validation logic for modifications +* **ToolRead Schema** - Extended to expose passthrough configuration in API responses + +### Documentation + +* REST Passthrough API configuration guide (see `docs/docs/using/rest-passthrough.md`) +* Updated tool configuration examples with passthrough fields +* Admin UI usage documentation for advanced REST tool configuration + +### Issues Closed + +**REST Integration:** +- Closes #746 - REST Passthrough API configuration fields + +--- + ## [0.8.0] - 2025-10-07 - Advanced OAuth, Plugin Ecosystem & MCP Registry ### Overview diff --git a/docs/docs/using/.pages b/docs/docs/using/.pages index 677b287bf..90ad3e694 100644 --- a/docs/docs/using/.pages +++ b/docs/docs/using/.pages @@ -5,6 +5,7 @@ nav: - reverse-proxy.md - multi-auth-headers.md - tool-annotations.md + - rest-passthrough.md - Clients: clients - Agents: agents - Servers: servers diff --git a/docs/docs/using/rest-passthrough.md b/docs/docs/using/rest-passthrough.md new file mode 100644 index 000000000..3bc76d83d --- /dev/null +++ b/docs/docs/using/rest-passthrough.md @@ -0,0 +1,594 @@ +# REST Passthrough Configuration + +Advanced configuration options for REST tools, enabling fine-grained control over request routing, header/query mapping, timeouts, security policies, and plugin chains. + +## Overview + +REST passthrough fields provide comprehensive control over how REST tools interact with upstream APIs: + +- **URL Mapping**: Automatic extraction of base URLs and path templates from tool URLs +- **Dynamic Parameters**: Query and header mapping for request customization +- **Security Controls**: Host allowlists and timeout configurations +- **Plugin Integration**: Pre and post-request plugin chain support +- **Flexible Configuration**: Per-tool timeout and exposure settings + +## Passthrough Fields + +| Field | Type | Description | Default | +|-------|------|-------------|---------| +| `base_url` | `string` | Base URL for REST passthrough (auto-extracted from `url`) | - | +| `path_template` | `string` | Path template for URL construction (auto-extracted) | - | +| `query_mapping` | `object` | JSON mapping for query parameters | `{}` | +| `header_mapping` | `object` | JSON mapping for headers | `{}` | +| `timeout_ms` | `integer` | Request timeout in milliseconds | `20000` | +| `expose_passthrough` | `boolean` | Enable/disable passthrough endpoint | `true` | +| `allowlist` | `array` | Allowed upstream hosts/schemes | `[]` | +| `plugin_chain_pre` | `array` | Pre-request plugin chain | `[]` | +| `plugin_chain_post` | `array` | Post-request plugin chain | `[]` | + +## Field Details + +### Base URL & Path Template + +When creating a REST tool, the `base_url` and `path_template` are automatically extracted from the `url` field: + +**Input:** +```json +{ + "url": "https://api.example.com/v1/users/{user_id}" +} +``` + +**Auto-extracted:** +```json +{ + "base_url": "https://api.example.com", + "path_template": "/v1/users/{user_id}" +} +``` + +**Validation:** +- `base_url` must have a valid scheme (http/https) and netloc +- `path_template` must start with `/` + +### Query Mapping + +Map tool parameters to query string parameters: + +```json +{ + "query_mapping": { + "userId": "user_id", + "includeDetails": "include_details", + "format": "response_format" + } +} +``` + +**Example Usage:** +When a tool is invoked with: +```json +{ + "userId": "123", + "includeDetails": true, + "format": "json" +} +``` + +The gateway constructs: +``` +GET https://api.example.com/endpoint?user_id=123&include_details=true&response_format=json +``` + +### Header Mapping + +Map tool parameters to HTTP headers: + +```json +{ + "header_mapping": { + "apiKey": "X-API-Key", + "clientId": "X-Client-ID", + "requestId": "X-Request-ID" + } +} +``` + +**Example Usage:** +When a tool is invoked with: +```json +{ + "apiKey": "secret123", + "clientId": "client-456", + "requestId": "req-789" +} +``` + +The gateway sends: +```http +X-API-Key: secret123 +X-Client-ID: client-456 +X-Request-ID: req-789 +``` + +### Timeout Configuration + +Set per-tool timeout in milliseconds: + +```json +{ + "timeout_ms": 30000 +} +``` + +**Default Behavior:** +- For REST tools with `expose_passthrough: true`: `20000ms` (20 seconds) +- For other integration types: No default timeout + +**Validation:** +- Must be a positive integer +- Recommended range: `5000-60000ms` (5-60 seconds) + +### Expose Passthrough + +Control whether the passthrough endpoint is exposed: + +```json +{ + "expose_passthrough": false +} +``` + +**Use Cases:** +- `true` (default): Expose passthrough endpoint for direct REST invocation +- `false`: Hide passthrough, only allow invocation through gateway + +### Allowlist + +Restrict upstream hosts/schemes that tools can connect to: + +```json +{ + "allowlist": [ + "api.example.com", + "https://secure.api.com", + "internal.company.net:8080" + ] +} +``` + +**Validation:** +- Each entry must match hostname regex: `^(https?://)?([a-zA-Z0-9.-]+)(:[0-9]+)?$` +- Supports optional scheme prefix and port suffix + +**Security Benefits:** +- Prevents SSRF (Server-Side Request Forgery) attacks +- Restricts tool access to approved endpoints only +- Enforces organizational security policies + +### Plugin Chains + +Configure pre and post-request plugin processing: + +```json +{ + "plugin_chain_pre": ["deny_filter", "rate_limit", "pii_filter"], + "plugin_chain_post": ["response_shape", "regex_filter"] +} +``` + +**Allowed Plugins:** +- `deny_filter` - Block requests matching deny patterns +- `rate_limit` - Rate limiting enforcement +- `pii_filter` - PII detection and filtering +- `response_shape` - Response transformation +- `regex_filter` - Regex-based content filtering +- `resource_filter` - Resource access control + +**Execution Order:** +1. **Pre-request plugins** (`plugin_chain_pre`) execute before the REST call +2. REST call to upstream API +3. **Post-request plugins** (`plugin_chain_post`) execute after receiving response + +## Setting Passthrough Fields via Admin UI + +### Using the Advanced Button + +1. Navigate to **Tools** section in the Admin UI +2. Click **Add Tool** or **Edit** on an existing tool +3. Select **Integration Type**: `REST` +4. Enter the **URL** (e.g., `https://api.example.com/v1/users`) +5. Click **Advanced: Add Passthrough** button +6. Configure passthrough fields in the expanded section: + - **Query Mapping (JSON)**: `{"userId": "user_id"}` + - **Header Mapping (JSON)**: `{"apiKey": "X-API-Key"}` + - **Timeout MS**: `30000` + - **Expose Passthrough**: `true` or `false` + - **Allowlist**: `["api.example.com"]` + - **Plugin Chain Pre**: `["rate_limit", "pii_filter"]` + - **Plugin Chain Post**: `["response_shape"]` +7. Click **Save** + +## Setting Passthrough Fields via API + +### Complete Example with All Fields + +```bash +curl -X POST /tools \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOKEN" \ + -d '{ + "name": "user-management-api", + "integration_type": "REST", + "request_type": "GET", + "url": "https://api.example.com/v1/users/{user_id}", + "description": "Fetch user information from external API", + "query_mapping": { + "includeMetadata": "include_metadata", + "fields": "response_fields" + }, + "header_mapping": { + "apiKey": "X-API-Key", + "tenantId": "X-Tenant-ID" + }, + "timeout_ms": 25000, + "expose_passthrough": true, + "allowlist": [ + "api.example.com", + "https://backup-api.example.com" + ], + "plugin_chain_pre": ["rate_limit", "pii_filter"], + "plugin_chain_post": ["response_shape"] + }' +``` + +### Minimal Example (Defaults Applied) + +```bash +curl -X POST /tools \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOKEN" \ + -d '{ + "name": "simple-rest-tool", + "integration_type": "REST", + "request_type": "POST", + "url": "https://api.example.com/v1/create" + }' +``` + +**Auto-applied Defaults:** +- `base_url`: `https://api.example.com` (extracted) +- `path_template`: `/v1/create` (extracted) +- `timeout_ms`: `20000` (default for REST passthrough) +- `expose_passthrough`: `true` +- `query_mapping`: `{}` +- `header_mapping`: `{}` +- `allowlist`: `[]` +- `plugin_chain_pre`: `[]` +- `plugin_chain_post`: `[]` + +### Updating Passthrough Fields + +```bash +curl -X PUT /tools/{tool_id} \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOKEN" \ + -d '{ + "timeout_ms": 30000, + "allowlist": ["api.example.com", "api2.example.com"], + "plugin_chain_pre": ["rate_limit", "deny_filter"] + }' +``` + +## Common Use Cases + +### 1. API Key Authentication via Headers + +```json +{ + "name": "external-api-with-auth", + "url": "https://api.service.com/data", + "header_mapping": { + "apiKey": "X-API-Key", + "apiSecret": "X-API-Secret" + }, + "allowlist": ["api.service.com"] +} +``` + +### 2. Search API with Query Parameters + +```json +{ + "name": "search-api", + "url": "https://search.example.com/query", + "query_mapping": { + "searchTerm": "q", + "maxResults": "limit", + "pageNumber": "page" + }, + "timeout_ms": 15000 +} +``` + +### 3. High-Security API with Plugins + +```json +{ + "name": "sensitive-data-api", + "url": "https://secure-api.company.internal/data", + "allowlist": ["secure-api.company.internal"], + "plugin_chain_pre": ["deny_filter", "rate_limit", "pii_filter"], + "plugin_chain_post": ["response_shape", "pii_filter"], + "timeout_ms": 10000 +} +``` + +### 4. Multi-Tenant API with Dynamic Headers + +```json +{ + "name": "multi-tenant-service", + "url": "https://api.saas.com/v2/tenants/{tenant_id}/resources", + "header_mapping": { + "tenantApiKey": "X-Tenant-API-Key", + "organizationId": "X-Organization-ID" + }, + "query_mapping": { + "includeArchived": "include_archived" + }, + "timeout_ms": 20000 +} +``` + +### 5. Rate-Limited Public API + +```json +{ + "name": "public-api-with-limits", + "url": "https://public-api.example.com/v1/data", + "plugin_chain_pre": ["rate_limit"], + "timeout_ms": 30000, + "allowlist": ["public-api.example.com"] +} +``` + +## Validation Rules + +### Enforced Constraints + +1. **Integration Type Restriction**: Passthrough fields only valid for `integration_type: "REST"` + ```json + // ❌ Invalid - passthrough fields on non-REST tool + { + "integration_type": "MCP", + "query_mapping": {...} // Error! + } + ``` + +2. **Base URL Format**: Must include scheme and netloc + ```json + // ✅ Valid + "base_url": "https://api.example.com" + + // ❌ Invalid + "base_url": "api.example.com" // Missing scheme + ``` + +3. **Path Template Format**: Must start with `/` + ```json + // ✅ Valid + "path_template": "/v1/users" + + // ❌ Invalid + "path_template": "v1/users" // Missing leading slash + ``` + +4. **Timeout Range**: Must be positive integer + ```json + // ✅ Valid + "timeout_ms": 25000 + + // ❌ Invalid + "timeout_ms": -1000 // Negative value + ``` + +5. **Plugin Validation**: Only allowed plugins + ```json + // ✅ Valid + "plugin_chain_pre": ["rate_limit", "pii_filter"] + + // ❌ Invalid + "plugin_chain_pre": ["unknown_plugin"] // Not in allowed list + ``` + +## Security Best Practices + +### 1. Always Use Allowlists for Production + +```json +{ + "allowlist": [ + "api.production.com", + "backup.production.com" + ] +} +``` + +**Benefits:** +- Prevents SSRF attacks +- Enforces approved endpoints only +- Auditable security policy + +### 2. Set Appropriate Timeouts + +```json +{ + "timeout_ms": 15000 // 15 seconds for most APIs +} +``` + +**Guidelines:** +- Fast APIs: `5000-10000ms` +- Standard APIs: `15000-25000ms` +- Batch/Long-running: `30000-60000ms` + +### 3. Use PII Filtering for Sensitive Data + +```json +{ + "plugin_chain_pre": ["pii_filter"], + "plugin_chain_post": ["pii_filter"] +} +``` + +**Protects:** +- Personally identifiable information +- Credit card numbers +- Social security numbers +- Email addresses + +### 4. Rate Limit External APIs + +```json +{ + "plugin_chain_pre": ["rate_limit"] +} +``` + +**Prevents:** +- API quota exhaustion +- DDoS to upstream services +- Unexpected billing charges + +### 5. Validate Response Shapes + +```json +{ + "plugin_chain_post": ["response_shape"] +} +``` + +**Ensures:** +- Consistent response formats +- Expected data structures +- Type safety + +## Troubleshooting + +### Common Issues + +#### Issue: "Field 'query_mapping' is only allowed for integration_type 'REST'" + +**Solution:** Ensure `integration_type: "REST"` is set: +```json +{ + "integration_type": "REST", + "query_mapping": {...} +} +``` + +#### Issue: "base_url must be a valid URL with scheme and netloc" + +**Solution:** Include `https://` or `http://` prefix: +```json +{ + "base_url": "https://api.example.com" // Not "api.example.com" +} +``` + +#### Issue: "path_template must start with '/'" + +**Solution:** Add leading slash: +```json +{ + "path_template": "/v1/users" // Not "v1/users" +} +``` + +#### Issue: "Unknown plugin: custom_plugin" + +**Solution:** Use only allowed plugins: +```json +{ + "plugin_chain_pre": [ + "deny_filter", + "rate_limit", + "pii_filter", + "response_shape", + "regex_filter", + "resource_filter" + ] +} +``` + +#### Issue: "timeout_ms must be a positive integer" + +**Solution:** Provide valid positive number: +```json +{ + "timeout_ms": 20000 // Not -1, 0, or non-integer +} +``` + +## Migration from Previous Versions + +### If you have existing REST tools without passthrough fields: + +**Before (v0.8.0):** +```json +{ + "name": "my-rest-tool", + "integration_type": "REST", + "url": "https://api.example.com/v1/users" +} +``` + +**After (v0.9.0):** +```json +{ + "name": "my-rest-tool", + "integration_type": "REST", + "url": "https://api.example.com/v1/users", + // Auto-extracted fields: + "base_url": "https://api.example.com", + "path_template": "/v1/users", + // Auto-applied defaults: + "timeout_ms": 20000, + "expose_passthrough": true +} +``` + +**No action required** - existing tools will continue to work with auto-applied defaults. + +## API Reference + +### Tool Schema with Passthrough Fields + +```typescript +interface ToolCreate { + name: string; + integration_type: "REST" | "MCP" | "A2A"; + request_type: string; + url: string; + description?: string; + + // REST Passthrough Fields (only for integration_type: "REST") + base_url?: string; // Auto-extracted from url + path_template?: string; // Auto-extracted from url + query_mapping?: object; // Default: {} + header_mapping?: object; // Default: {} + timeout_ms?: number; // Default: 20000 for REST + expose_passthrough?: boolean; // Default: true + allowlist?: string[]; // Default: [] + plugin_chain_pre?: string[]; // Default: [] + plugin_chain_post?: string[]; // Default: [] +} +``` + +## See Also + +- [Tool Annotations](./tool-annotations.md) - Behavioral hints for tools +- [Plugin Framework](../plugins/index.md) - Plugin development and usage +- [Multi-Auth Headers](./multi-auth-headers.md) - Authentication header configuration +- [Reverse Proxy](./reverse-proxy.md) - Reverse proxy configuration diff --git a/mcpgateway/admin.py b/mcpgateway/admin.py index 0564d07d0..5ae92503a 100644 --- a/mcpgateway/admin.py +++ b/mcpgateway/admin.py @@ -5102,6 +5102,13 @@ async def admin_add_tool( "visibility": visibility, "team_id": team_id, "owner_email": user_email, + "query_mapping": json.loads(form.get("query_mapping") or "{}"), + "header_mapping": json.loads(form.get("header_mapping") or "{}"), + "timeout_ms": int(form.get("timeout_ms")) if form.get("timeout_ms") and form.get("timeout_ms").strip() else None, + "expose_passthrough": form.get("expose_passthrough", "true"), + "allowlist": json.loads(form.get("allowlist") or "[]"), + "plugin_chain_pre": json.loads(form.get("plugin_chain_pre") or "[]"), + "plugin_chain_post": json.loads(form.get("plugin_chain_post") or "[]"), } LOGGER.debug(f"Tool data built: {tool_data}") try: diff --git a/mcpgateway/alembic/versions/8a2934be50c0_rest_pass_api_fld_tools.py b/mcpgateway/alembic/versions/8a2934be50c0_rest_pass_api_fld_tools.py new file mode 100644 index 000000000..2f4b2437c --- /dev/null +++ b/mcpgateway/alembic/versions/8a2934be50c0_rest_pass_api_fld_tools.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +"""rest_pass_api_fld_tools + +Revision ID: 8a2934be50c0 +Revises: 9aaa90ad26d9 +Create Date: 2025-10-17 12:19:39.576193 + +""" + +# Standard +from typing import Sequence, Union + +# Third-Party +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "8a2934be50c0" +down_revision: Union[str, Sequence[str], None] = "9aaa90ad26d9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # Add Passthrough REST fields to tools table + op.add_column("tools", sa.Column("base_url", sa.String(), nullable=True)) + op.add_column("tools", sa.Column("path_template", sa.String(), nullable=True)) + op.add_column("tools", sa.Column("query_mapping", sa.JSON(), nullable=True)) + op.add_column("tools", sa.Column("header_mapping", sa.JSON(), nullable=True)) + op.add_column("tools", sa.Column("timeout_ms", sa.Integer(), nullable=True)) + op.add_column("tools", sa.Column("expose_passthrough", sa.Boolean(), nullable=False, server_default="1")) + op.add_column("tools", sa.Column("allowlist", sa.JSON(), nullable=True)) + op.add_column("tools", sa.Column("plugin_chain_pre", sa.JSON(), nullable=True)) + op.add_column("tools", sa.Column("plugin_chain_post", sa.JSON(), nullable=True)) + + +def downgrade() -> None: + """Downgrade schema.""" + # Remove Passthrough REST fields from tools table + op.drop_column("tools", "plugin_chain_post") + op.drop_column("tools", "plugin_chain_pre") + op.drop_column("tools", "allowlist") + op.drop_column("tools", "expose_passthrough") + op.drop_column("tools", "timeout_ms") + op.drop_column("tools", "header_mapping") + op.drop_column("tools", "query_mapping") + op.drop_column("tools", "path_template") + op.drop_column("tools", "base_url") diff --git a/mcpgateway/db.py b/mcpgateway/db.py index 91dd02e1d..91853c292 100644 --- a/mcpgateway/db.py +++ b/mcpgateway/db.py @@ -1603,6 +1603,17 @@ class Tool(Base): custom_name_slug: Mapped[Optional[str]] = mapped_column(String(255), nullable=False) display_name: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + # Passthrough REST fields + base_url: Mapped[Optional[str]] = mapped_column(String, nullable=True) + path_template: Mapped[Optional[str]] = mapped_column(String, nullable=True) + query_mapping: Mapped[Optional[Dict[str, Any]]] = mapped_column(JSON, nullable=True) + header_mapping: Mapped[Optional[Dict[str, Any]]] = mapped_column(JSON, nullable=True) + timeout_ms: Mapped[Optional[int]] = mapped_column(Integer, nullable=True, default=None) + expose_passthrough: Mapped[bool] = mapped_column(Boolean, default=True) + allowlist: Mapped[Optional[List[str]]] = mapped_column(JSON, nullable=True) + plugin_chain_pre: Mapped[Optional[List[str]]] = mapped_column(JSON, nullable=True) + plugin_chain_post: Mapped[Optional[List[str]]] = mapped_column(JSON, nullable=True) + # Federation relationship with a local gateway gateway_id: Mapped[Optional[str]] = mapped_column(ForeignKey("gateways.id")) # gateway_slug: Mapped[Optional[str]] = mapped_column(ForeignKey("gateways.slug")) diff --git a/mcpgateway/schemas.py b/mcpgateway/schemas.py index 037a04db1..368e8508a 100644 --- a/mcpgateway/schemas.py +++ b/mcpgateway/schemas.py @@ -27,6 +27,7 @@ import logging import re from typing import Any, Dict, List, Literal, Optional, Self, Union +from urllib.parse import urlparse # Third-Party from pydantic import AnyHttpUrl, BaseModel, ConfigDict, EmailStr, Field, field_serializer, field_validator, model_validator, ValidationInfo @@ -422,6 +423,17 @@ class ToolCreate(BaseModel): owner_email: Optional[str] = Field(None, description="Email of the tool owner") visibility: Optional[str] = Field(default="public", description="Visibility level (private, team, public)") + # Passthrough REST fields + base_url: Optional[str] = Field(None, description="Base URL for REST passthrough") + path_template: Optional[str] = Field(None, description="Path template for REST passthrough") + query_mapping: Optional[Dict[str, Any]] = Field(None, description="Query mapping for REST passthrough") + header_mapping: Optional[Dict[str, Any]] = Field(None, description="Header mapping for REST passthrough") + timeout_ms: Optional[int] = Field(default=None, description="Timeout in milliseconds for REST passthrough (20000 if integration_type='REST', else None)") + expose_passthrough: Optional[bool] = Field(True, description="Expose passthrough endpoint for this tool") + allowlist: Optional[List[str]] = Field(None, description="Allowed upstream hosts/schemes for passthrough") + plugin_chain_pre: Optional[List[str]] = Field(None, description="Pre-plugin chain for passthrough") + plugin_chain_post: Optional[List[str]] = Field(None, description="Post-plugin chain for passthrough") + @field_validator("tags") @classmethod def validate_tags(cls, v: Optional[List[str]]) -> List[str]: @@ -753,6 +765,184 @@ def prevent_manual_mcp_creation(cls, values: Dict[str, Any]) -> Dict[str, Any]: raise ValueError("Cannot manually create A2A tools. Add A2A agents via the A2A interface - tools will be auto-created when agents are associated with servers.") return values + @model_validator(mode="before") + @classmethod + def enforce_passthrough_fields_for_rest(cls, values: Dict[str, Any]) -> Dict[str, Any]: + """ + Enforce that passthrough REST fields are only set for integration_type 'REST'. + If any passthrough field is set for non-REST, raise ValueError. + + Args: + values (Dict[str, Any]): The input values to validate. + + Returns: + Dict[str, Any]: The validated values. + + Raises: + ValueError: If passthrough fields are set for non-REST integration_type. + """ + passthrough_fields = ["base_url", "path_template", "query_mapping", "header_mapping", "timeout_ms", "expose_passthrough", "allowlist", "plugin_chain_pre", "plugin_chain_post"] + integration_type = values.get("integration_type") + if integration_type != "REST": + for field in passthrough_fields: + if field in values and values[field] not in (None, [], {}): + raise ValueError(f"Field '{field}' is only allowed for integration_type 'REST'.") + return values + + @model_validator(mode="before") + @classmethod + def extract_base_url_and_path_template(cls, values: dict) -> dict: + """ + Only for integration_type 'REST': + If 'url' is provided, extract 'base_url' and 'path_template'. + Ensures path_template starts with a single '/'. + + Args: + values (dict): The input values to process. + + Returns: + dict: The updated values with base_url and path_template if applicable. + """ + integration_type = values.get("integration_type") + if integration_type != "REST": + # Only process for REST, skip for others + return values + url = values.get("url") + if url: + parsed = urlparse(str(url)) + base_url = f"{parsed.scheme}://{parsed.netloc}" + path_template = parsed.path + # Ensure path_template starts with a single '/' + if path_template: + path_template = "/" + path_template.lstrip("/") + if not values.get("base_url"): + values["base_url"] = base_url + if not values.get("path_template"): + values["path_template"] = path_template + return values + + @field_validator("base_url") + @classmethod + def validate_base_url(cls, v): + """ + Validate that base_url is a valid URL with scheme and netloc. + + Args: + v (str): The base_url value to validate. + + Returns: + str: The validated base_url value. + + Raises: + ValueError: If base_url is not a valid URL. + """ + if v is None: + return v + parsed = urlparse(str(v)) + if not parsed.scheme or not parsed.netloc: + raise ValueError("base_url must be a valid URL with scheme and netloc") + return v + + @field_validator("path_template") + @classmethod + def validate_path_template(cls, v): + """ + Validate that path_template starts with '/'. + + Args: + v (str): The path_template value to validate. + + Returns: + str: The validated path_template value. + + Raises: + ValueError: If path_template does not start with '/'. + """ + if v and not str(v).startswith("/"): + raise ValueError("path_template must start with '/'") + return v + + @field_validator("timeout_ms") + @classmethod + def validate_timeout_ms(cls, v): + """ + Validate that timeout_ms is a positive integer. + + Args: + v (int): The timeout_ms value to validate. + + Returns: + int: The validated timeout_ms value. + + Raises: + ValueError: If timeout_ms is not a positive integer. + """ + if v is not None and v <= 0: + raise ValueError("timeout_ms must be a positive integer") + return v + + @field_validator("allowlist") + @classmethod + def validate_allowlist(cls, v): + """ + Validate that allowlist is a list and each entry is a valid host or scheme string. + + Args: + v (List[str]): The allowlist to validate. + + Returns: + List[str]: The validated allowlist. + + Raises: + ValueError: If allowlist is not a list or any entry is not a valid host/scheme string. + """ + if v is None: + return None + if not isinstance(v, list): + raise ValueError("allowlist must be a list of host/scheme strings") + hostname_regex = re.compile(r"^(https?://)?([a-zA-Z0-9.-]+)(:[0-9]+)?$") + for host in v: + if not isinstance(host, str): + raise ValueError(f"Invalid type in allowlist: {host} (must be str)") + if not hostname_regex.match(host): + raise ValueError(f"Invalid host/scheme in allowlist: {host}") + return v + + @field_validator("plugin_chain_pre", "plugin_chain_post") + @classmethod + def validate_plugin_chain(cls, v): + """ + Validate that each plugin in the chain is allowed. + + Args: + v (List[str]): The plugin chain to validate. + + Returns: + List[str]: The validated plugin chain. + + Raises: + ValueError: If any plugin is not in the allowed set. + """ + allowed_plugins = {"deny_filter", "rate_limit", "pii_filter", "response_shape", "regex_filter", "resource_filter"} + if v is None: + return v + for plugin in v: + if plugin not in allowed_plugins: + raise ValueError(f"Unknown plugin: {plugin}") + return v + + @model_validator(mode="after") + def handle_timeout_ms_defaults(self): + """Handle timeout_ms defaults based on integration_type and expose_passthrough. + + Returns: + self: The validated model instance with timeout_ms potentially set to default. + """ + # If timeout_ms is None and we have REST with passthrough, set default + if self.timeout_ms is None and self.integration_type == "REST" and getattr(self, "expose_passthrough", True): + self.timeout_ms = 20000 + return self + class ToolUpdate(BaseModelWithConfigDict): """Schema for updating an existing tool. @@ -777,6 +967,17 @@ class ToolUpdate(BaseModelWithConfigDict): tags: Optional[List[str]] = Field(None, description="Tags for categorizing the tool") visibility: Optional[str] = Field(default="public", description="Visibility level: private, team, or public") + # Passthrough REST fields + base_url: Optional[str] = Field(None, description="Base URL for REST passthrough") + path_template: Optional[str] = Field(None, description="Path template for REST passthrough") + query_mapping: Optional[Dict[str, Any]] = Field(None, description="Query mapping for REST passthrough") + header_mapping: Optional[Dict[str, Any]] = Field(None, description="Header mapping for REST passthrough") + timeout_ms: Optional[int] = Field(default=None, description="Timeout in milliseconds for REST passthrough (20000 if integration_type='REST', else None)") + expose_passthrough: Optional[bool] = Field(True, description="Expose passthrough endpoint for this tool") + allowlist: Optional[List[str]] = Field(None, description="Allowed upstream hosts/schemes for passthrough") + plugin_chain_pre: Optional[List[str]] = Field(None, description="Pre-plugin chain for passthrough") + plugin_chain_post: Optional[List[str]] = Field(None, description="Post-plugin chain for passthrough") + @field_validator("tags") @classmethod def validate_tags(cls, v: Optional[List[str]]) -> List[str]: @@ -1012,6 +1213,146 @@ def prevent_manual_mcp_update(cls, values: Dict[str, Any]) -> Dict[str, Any]: raise ValueError("Cannot update tools to A2A integration type. A2A tools are managed by the A2A service.") return values + @model_validator(mode="before") + @classmethod + def extract_base_url_and_path_template(cls, values: dict) -> dict: + """ + If 'integration_type' is 'REST' and 'url' is provided, extract 'base_url' and 'path_template'. + Ensures path_template starts with a single '/'. + + Args: + values (dict): The input values to process. + + Returns: + dict: The updated values with base_url and path_template if applicable. + """ + integration_type = values.get("integration_type") + url = values.get("url") + if integration_type == "REST" and url: + parsed = urlparse(str(url)) + base_url = f"{parsed.scheme}://{parsed.netloc}" + path_template = parsed.path + # Ensure path_template starts with a single '/' + if path_template and not path_template.startswith("/"): + path_template = "/" + path_template.lstrip("/") + elif path_template: + path_template = "/" + path_template.lstrip("/") + if not values.get("base_url"): + values["base_url"] = base_url + if not values.get("path_template"): + values["path_template"] = path_template + return values + + @field_validator("base_url") + @classmethod + def validate_base_url(cls, v): + """ + Validate that base_url is a valid URL with scheme and netloc. + + Args: + v (str): The base_url value to validate. + + Returns: + str: The validated base_url value. + + Raises: + ValueError: If base_url is not a valid URL. + """ + if v is None: + return v + parsed = urlparse(str(v)) + if not parsed.scheme or not parsed.netloc: + raise ValueError("base_url must be a valid URL with scheme and netloc") + return v + + @field_validator("path_template") + @classmethod + def validate_path_template(cls, v): + """ + Validate that path_template starts with '/'. + + Args: + v (str): The path_template value to validate. + + Returns: + str: The validated path_template value. + + Raises: + ValueError: If path_template does not start with '/'. + """ + if v and not str(v).startswith("/"): + raise ValueError("path_template must start with '/'") + return v + + @field_validator("timeout_ms") + @classmethod + def validate_timeout_ms(cls, v): + """ + Validate that timeout_ms is a positive integer. + + Args: + v (int): The timeout_ms value to validate. + + Returns: + int: The validated timeout_ms value. + + Raises: + ValueError: If timeout_ms is not a positive integer. + """ + if v is not None and v <= 0: + raise ValueError("timeout_ms must be a positive integer") + return v + + @field_validator("allowlist") + @classmethod + def validate_allowlist(cls, v): + """ + Validate that allowlist is a list and each entry is a valid host or scheme string. + + Args: + v (List[str]): The allowlist to validate. + + Returns: + List[str]: The validated allowlist. + + Raises: + ValueError: If allowlist is not a list or any entry is not a valid host/scheme string. + """ + if v is None: + return None + if not isinstance(v, list): + raise ValueError("allowlist must be a list of host/scheme strings") + hostname_regex = re.compile(r"^(https?://)?([a-zA-Z0-9.-]+)(:[0-9]+)?$") + for host in v: + if not isinstance(host, str): + raise ValueError(f"Invalid type in allowlist: {host} (must be str)") + if not hostname_regex.match(host): + raise ValueError(f"Invalid host/scheme in allowlist: {host}") + return v + + @field_validator("plugin_chain_pre", "plugin_chain_post") + @classmethod + def validate_plugin_chain(cls, v): + """ + Validate that each plugin in the chain is allowed. + + Args: + v (List[str]): The plugin chain to validate. + + Returns: + List[str]: The validated plugin chain. + + Raises: + ValueError: If any plugin is not in the allowed set. + """ + allowed_plugins = {"deny_filter", "rate_limit", "pii_filter", "response_shape", "regex_filter", "resource_filter"} + if v is None: + return v + for plugin in v: + if plugin not in allowed_plugins: + raise ValueError(f"Unknown plugin: {plugin}") + return v + class ToolRead(BaseModelWithConfigDict): """Schema for reading tool information. @@ -1074,6 +1415,17 @@ class ToolRead(BaseModelWithConfigDict): owner_email: Optional[str] = Field(None, description="Email of the user who owns this resource") visibility: Optional[str] = Field(default="public", description="Visibility level: private, team, or public") + # Passthrough REST fields + base_url: Optional[str] = Field(None, description="Base URL for REST passthrough") + path_template: Optional[str] = Field(None, description="Path template for REST passthrough") + query_mapping: Optional[Dict[str, Any]] = Field(None, description="Query mapping for REST passthrough") + header_mapping: Optional[Dict[str, Any]] = Field(None, description="Header mapping for REST passthrough") + timeout_ms: Optional[int] = Field(20000, description="Timeout in milliseconds for REST passthrough") + expose_passthrough: Optional[bool] = Field(True, description="Expose passthrough endpoint for this tool") + allowlist: Optional[List[str]] = Field(None, description="Allowed upstream hosts/schemes for passthrough") + plugin_chain_pre: Optional[List[str]] = Field(None, description="Pre-plugin chain for passthrough") + plugin_chain_post: Optional[List[str]] = Field(None, description="Post-plugin chain for passthrough") + class ToolInvocation(BaseModelWithConfigDict): """Schema for tool invocation requests. diff --git a/mcpgateway/services/tool_service.py b/mcpgateway/services/tool_service.py index 05414b2b6..90c806a3d 100644 --- a/mcpgateway/services/tool_service.py +++ b/mcpgateway/services/tool_service.py @@ -486,6 +486,16 @@ async def register_tool( team_id=team_id, owner_email=owner_email or created_by, visibility=visibility, + # passthrough REST tools fields + base_url=tool.base_url if tool.integration_type == "REST" else None, + path_template=tool.path_template if tool.integration_type == "REST" else None, + query_mapping=tool.query_mapping if tool.integration_type == "REST" else None, + header_mapping=tool.header_mapping if tool.integration_type == "REST" else None, + timeout_ms=tool.timeout_ms if tool.integration_type == "REST" else None, + expose_passthrough=(tool.expose_passthrough if tool.integration_type == "REST" and tool.expose_passthrough is not None else True) if tool.integration_type == "REST" else None, + allowlist=tool.allowlist if tool.integration_type == "REST" else None, + plugin_chain_pre=tool.plugin_chain_pre if tool.integration_type == "REST" else None, + plugin_chain_post=tool.plugin_chain_post if tool.integration_type == "REST" else None, ) db.add(db_tool) diff --git a/mcpgateway/static/admin.js b/mcpgateway/static/admin.js index 1c0fd1746..8a5fc75ee 100644 --- a/mcpgateway/static/admin.js +++ b/mcpgateway/static/admin.js @@ -1,3 +1,89 @@ +// Add three fields to passthrough section on Advanced button click +function handleAddPassthrough() { + const passthroughContainer = safeGetElement("passthrough-container"); + if (!passthroughContainer) { + console.error("Passthrough container not found"); + return; + } + + // Toggle visibility + if ( + passthroughContainer.style.display === "none" || + passthroughContainer.style.display === "" + ) { + passthroughContainer.style.display = "block"; + // Add fields only if not already present + if (!document.getElementById("query-mapping-field")) { + const queryDiv = document.createElement("div"); + queryDiv.className = "mb-4"; + queryDiv.innerHTML = ` + + + `; + passthroughContainer.appendChild(queryDiv); + } + if (!document.getElementById("header-mapping-field")) { + const headerDiv = document.createElement("div"); + headerDiv.className = "mb-4"; + headerDiv.innerHTML = ` + + + `; + passthroughContainer.appendChild(headerDiv); + } + if (!document.getElementById("timeout-ms-field")) { + const timeoutDiv = document.createElement("div"); + timeoutDiv.className = "mb-4"; + timeoutDiv.innerHTML = ` + + + `; + passthroughContainer.appendChild(timeoutDiv); + } + if (!document.getElementById("expose-passthrough-field")) { + const exposeDiv = document.createElement("div"); + exposeDiv.className = "mb-4"; + exposeDiv.innerHTML = ` + + + `; + passthroughContainer.appendChild(exposeDiv); + } + if (!document.getElementById("allowlist-field")) { + const allowlistDiv = document.createElement("div"); + allowlistDiv.className = "mb-4"; + allowlistDiv.innerHTML = ` + + + `; + passthroughContainer.appendChild(allowlistDiv); + } + if (!document.getElementById("plugin-chain-pre-field")) { + const pluginPreDiv = document.createElement("div"); + pluginPreDiv.className = "mb-4"; + pluginPreDiv.innerHTML = ` + + + `; + passthroughContainer.appendChild(pluginPreDiv); + } + if (!document.getElementById("plugin-chain-post-field")) { + const pluginPostDiv = document.createElement("div"); + pluginPostDiv.className = "mb-4"; + pluginPostDiv.innerHTML = ` + + + `; + passthroughContainer.appendChild(pluginPostDiv); + } + } else { + passthroughContainer.style.display = "none"; + } +} + // Make URL field read-only for integration type MCP function updateEditToolUrl() { const editTypeField = document.getElementById("edit-tool-type"); @@ -9099,6 +9185,11 @@ function setupFormHandlers() { paramButton.addEventListener("click", handleAddParameter); } + const passthroughButton = safeGetElement("add-passthrough-btn"); + if (passthroughButton) { + passthroughButton.addEventListener("click", handleAddPassthrough); + } + const serverForm = safeGetElement("add-server-form"); if (serverForm) { serverForm.addEventListener("submit", handleServerFormSubmit); diff --git a/mcpgateway/templates/admin.html b/mcpgateway/templates/admin.html index 904b2e606..44e52a5fe 100644 --- a/mcpgateway/templates/admin.html +++ b/mcpgateway/templates/admin.html @@ -3079,6 +3079,17 @@

> Add New Parameter + + +