diff --git a/docs/src/content/docs/reference/frontmatter-full.md b/docs/src/content/docs/reference/frontmatter-full.md index bef1bf7cd1..4c3c5b29c9 100644 --- a/docs/src/content/docs/reference/frontmatter-full.md +++ b/docs/src/content/docs/reference/frontmatter-full.md @@ -1096,23 +1096,22 @@ sandbox: domain: "localhost" # Plugin configuration for installing plugins before workflow execution. Supports -# both array format (list of repos) and object format (repos + custom token). +# array format (list of repos/plugin configs) and object format (repos + custom +# token). # (optional) # This field supports multiple formats (oneOf): -# Option 1: List of plugin repository slugs to install. Each plugin is installed -# using the engine's plugin installation command with default token resolution. +# Option 1: List of plugins to install. Each item can be either a repository slug +# string (e.g., 'org/repo') or an object with url and optional MCP configuration. plugins: [] - # Array items: Plugin repository slug in the format 'org/repo' (e.g., - # 'github/example-plugin') + # Array items: undefined -# Option 2: Plugin configuration with custom GitHub token. The custom token -# overrides the default token resolution chain. +# Option 2: Plugin configuration with custom GitHub token. Repos can be either +# strings or objects with MCP configuration. plugins: - # List of plugin repository slugs to install + # List of plugins to install. Each item can be either a repository slug string or + # an object with url and optional MCP configuration. repos: [] - # Array of Plugin repository slug in the format 'org/repo' (e.g., - # 'github/example-plugin') # Custom GitHub token expression to use for plugin installation. Overrides the # default cascading token resolution (GH_AW_PLUGINS_TOKEN -> GH_AW_GITHUB_TOKEN -> diff --git a/pkg/parser/import_processor.go b/pkg/parser/import_processor.go index f71ee766a9..954a2f697b 100644 --- a/pkg/parser/import_processor.go +++ b/pkg/parser/import_processor.go @@ -169,7 +169,7 @@ func processImportsFromFrontmatterWithManifestAndSource(frontmatter map[string]a var toolsBuilder strings.Builder var mcpServersBuilder strings.Builder var markdownBuilder strings.Builder // Only used for imports WITH inputs (compile-time substitution) - var importPaths []string // NEW: Track import paths for runtime-import macro generation + var importPaths []string // NEW: Track import paths for runtime-import macro generation var stepsBuilder strings.Builder var copilotSetupStepsBuilder strings.Builder // Track copilot-setup-steps.yml separately var runtimesBuilder strings.Builder @@ -317,7 +317,7 @@ func processImportsFromFrontmatterWithManifestAndSource(frontmatter map[string]a } else { // Has inputs - must inline for compile-time substitution log.Printf("Agent file has inputs - will be inlined instead of runtime-imported") - + // For agent files, extract markdown content (only when inputs are present) markdownContent, err := processIncludedFileWithVisited(item.fullPath, item.sectionName, false, visited) if err != nil { @@ -489,7 +489,7 @@ func processImportsFromFrontmatterWithManifestAndSource(frontmatter map[string]a } else { // Has inputs - must inline for compile-time substitution log.Printf("Import %s has inputs - will be inlined for compile-time substitution", importRelPath) - + // Extract markdown content from imported file (only for imports with inputs) markdownContent, err := processIncludedFileWithVisited(item.fullPath, item.sectionName, false, visited) if err != nil { @@ -584,15 +584,29 @@ func processImportsFromFrontmatterWithManifestAndSource(frontmatter map[string]a } // Extract plugins from imported file (merge into set to avoid duplicates) + // This now handles both simple string format and object format with MCP configs pluginsContent, err := extractPluginsFromContent(string(content)) if err == nil && pluginsContent != "" && pluginsContent != "[]" { - // Parse plugins JSON array - var importedPlugins []string - if jsonErr := json.Unmarshal([]byte(pluginsContent), &importedPlugins); jsonErr == nil { - for _, plugin := range importedPlugins { - if !pluginsSet[plugin] { - pluginsSet[plugin] = true - plugins = append(plugins, plugin) + // Parse plugins - can be array of strings or objects + var pluginsRaw []any + if jsonErr := json.Unmarshal([]byte(pluginsContent), &pluginsRaw); jsonErr == nil { + for _, item := range pluginsRaw { + // Handle string format: "org/repo" + if pluginStr, ok := item.(string); ok { + if !pluginsSet[pluginStr] { + pluginsSet[pluginStr] = true + plugins = append(plugins, pluginStr) + } + } else if pluginObj, ok := item.(map[string]any); ok { + // Handle object format: { "id": "org/repo", "mcp": {...} } + if idVal, hasID := pluginObj["id"]; hasID { + if pluginID, ok := idVal.(string); ok && !pluginsSet[pluginID] { + pluginsSet[pluginID] = true + plugins = append(plugins, pluginID) + // Note: MCP configs from imports are currently not merged + // They would need to be handled at a higher level in compiler_orchestrator_tools.go + } + } } } } @@ -639,7 +653,7 @@ func processImportsFromFrontmatterWithManifestAndSource(frontmatter map[string]a MergedSafeOutputs: safeOutputs, MergedSafeInputs: safeInputs, MergedMarkdown: markdownBuilder.String(), // Only imports WITH inputs (for compile-time substitution) - ImportPaths: importPaths, // Import paths for runtime-import macro generation + ImportPaths: importPaths, // Import paths for runtime-import macro generation MergedSteps: stepsBuilder.String(), CopilotSetupSteps: copilotSetupStepsBuilder.String(), MergedRuntimes: runtimesBuilder.String(), diff --git a/pkg/parser/schemas/main_workflow_schema.json b/pkg/parser/schemas/main_workflow_schema.json index 44c6ae1fc7..53bac0b92f 100644 --- a/pkg/parser/schemas/main_workflow_schema.json +++ b/pkg/parser/schemas/main_workflow_schema.json @@ -2483,9 +2483,20 @@ ] }, "plugins": { - "description": "Plugin configuration for installing plugins before workflow execution. Supports both array format (list of repos) and object format (repos + custom token).", + "description": "Plugin configuration for installing plugins before workflow execution. Supports array format (list of repos/plugin configs) and object format (repos + custom token).", "examples": [ ["github/copilot-plugin", "acme/custom-tools"], + [ + "github/simple-plugin", + { + "id": "github/mcp-plugin", + "mcp": { + "env": { + "API_KEY": "${{ secrets.API_KEY }}" + } + } + } + ], { "repos": ["github/copilot-plugin", "acme/custom-tools"], "github-token": "${{ secrets.CUSTOM_PLUGIN_TOKEN }}" @@ -2494,25 +2505,93 @@ "oneOf": [ { "type": "array", - "description": "List of plugin repository slugs to install. Each plugin is installed using the engine's plugin installation command with default token resolution.", + "description": "List of plugins to install. Each item can be either a repository slug string (e.g., 'org/repo') or an object with id and optional MCP configuration.", "items": { - "type": "string", - "pattern": "^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+$", - "description": "Plugin repository slug in the format 'org/repo' (e.g., 'github/example-plugin')" + "oneOf": [ + { + "type": "string", + "pattern": "^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+$", + "description": "Plugin repository slug in the format 'org/repo' (e.g., 'github/example-plugin')" + }, + { + "type": "object", + "description": "Plugin configuration with ID and optional MCP settings for environment variables", + "required": ["id"], + "properties": { + "id": { + "type": "string", + "pattern": "^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+$", + "description": "Plugin repository slug in the format 'org/repo' (e.g., 'github/example-plugin')" + }, + "mcp": { + "type": "object", + "description": "MCP server configuration for this plugin. When defined, the compiler scans the plugin's MCP JSON and mounts it in the gateway with the specified environment variables.", + "properties": { + "env": { + "type": "object", + "description": "Environment variables to pass when instantiating the MCP server. These variables are made available to the MCP server at runtime and can reference secrets using ${{ secrets.SECRET_NAME }} syntax.", + "additionalProperties": { + "type": "string" + }, + "examples": [ + { + "API_KEY": "${{ secrets.API_KEY }}", + "API_URL": "https://api.example.com" + } + ] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + ] } }, { "type": "object", - "description": "Plugin configuration with custom GitHub token. The custom token overrides the default token resolution chain.", + "description": "Plugin configuration with custom GitHub token. Repos can be either strings or objects with MCP configuration.", "required": ["repos"], "properties": { "repos": { "type": "array", - "description": "List of plugin repository slugs to install", + "description": "List of plugins to install. Each item can be either a repository slug string or an object with id and optional MCP configuration.", "items": { - "type": "string", - "pattern": "^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+$", - "description": "Plugin repository slug in the format 'org/repo' (e.g., 'github/example-plugin')" + "oneOf": [ + { + "type": "string", + "pattern": "^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+$", + "description": "Plugin repository slug in the format 'org/repo' (e.g., 'github/example-plugin')" + }, + { + "type": "object", + "description": "Plugin configuration with ID and optional MCP settings", + "required": ["id"], + "properties": { + "id": { + "type": "string", + "pattern": "^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+$", + "description": "Plugin repository slug in the format 'org/repo' (e.g., 'github/example-plugin')" + }, + "mcp": { + "type": "object", + "description": "MCP server configuration for this plugin", + "properties": { + "env": { + "type": "object", + "description": "Environment variables for the MCP server", + "additionalProperties": { + "type": "string" + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + ] } }, "github-token": { @@ -3537,7 +3616,7 @@ }, "description": "Toolsets to enable" }, - "url": { + "id": { "type": "string", "description": "URL for HTTP mode MCP servers" }, diff --git a/pkg/workflow/claude_engine.go b/pkg/workflow/claude_engine.go index f788859daf..3c1f0eda90 100644 --- a/pkg/workflow/claude_engine.go +++ b/pkg/workflow/claude_engine.go @@ -127,14 +127,14 @@ func (e *ClaudeEngine) GetInstallationSteps(workflowData *WorkflowData) []GitHub } // Add plugin installation steps after Claude CLI installation - if len(workflowData.Plugins) > 0 { - claudeLog.Printf("Adding plugin installation steps: %d plugins", len(workflowData.Plugins)) + if workflowData.PluginInfo != nil && len(workflowData.PluginInfo.Plugins) > 0 { + claudeLog.Printf("Adding plugin installation steps: %d plugins", len(workflowData.PluginInfo.Plugins)) // Use plugin-specific token if provided, otherwise use top-level github-token - tokenToUse := workflowData.PluginsToken + tokenToUse := workflowData.PluginInfo.CustomToken if tokenToUse == "" { tokenToUse = workflowData.GitHubToken } - pluginSteps := GeneratePluginInstallationSteps(workflowData.Plugins, "claude", tokenToUse) + pluginSteps := GeneratePluginInstallationSteps(workflowData.PluginInfo.Plugins, "claude", tokenToUse) steps = append(steps, pluginSteps...) } diff --git a/pkg/workflow/codex_engine.go b/pkg/workflow/codex_engine.go index d1aebde3d1..6f8d49e593 100644 --- a/pkg/workflow/codex_engine.go +++ b/pkg/workflow/codex_engine.go @@ -102,14 +102,14 @@ func (e *CodexEngine) GetInstallationSteps(workflowData *WorkflowData) []GitHubA } // Add plugin installation steps after Codex CLI installation - if len(workflowData.Plugins) > 0 { - codexEngineLog.Printf("Adding plugin installation steps: %d plugins", len(workflowData.Plugins)) + if workflowData.PluginInfo != nil && len(workflowData.PluginInfo.Plugins) > 0 { + codexEngineLog.Printf("Adding plugin installation steps: %d plugins", len(workflowData.PluginInfo.Plugins)) // Use plugin-specific token if provided, otherwise use top-level github-token - tokenToUse := workflowData.PluginsToken + tokenToUse := workflowData.PluginInfo.CustomToken if tokenToUse == "" { tokenToUse = workflowData.GitHubToken } - pluginSteps := GeneratePluginInstallationSteps(workflowData.Plugins, "codex", tokenToUse) + pluginSteps := GeneratePluginInstallationSteps(workflowData.PluginInfo.Plugins, "codex", tokenToUse) steps = append(steps, pluginSteps...) } diff --git a/pkg/workflow/compiler_orchestrator_tools.go b/pkg/workflow/compiler_orchestrator_tools.go index aefcd31606..964f8a9a8c 100644 --- a/pkg/workflow/compiler_orchestrator_tools.go +++ b/pkg/workflow/compiler_orchestrator_tools.go @@ -17,8 +17,7 @@ var orchestratorToolsLog = logger.New("workflow:compiler_orchestrator_tools") type toolsProcessingResult struct { tools map[string]any runtimes map[string]any - plugins []string - pluginsToken string + pluginInfo *PluginInfo // Consolidated plugin information toolsTimeout int toolsStartupTimeout int markdownContent string @@ -140,13 +139,20 @@ func (c *Compiler) processToolsAndMarkdown(result *parser.FrontmatterResult, cle } // Extract plugins from frontmatter - plugins, pluginsToken := extractPluginsFromFrontmatter(result.Frontmatter) - if len(plugins) > 0 { - orchestratorToolsLog.Printf("Extracted %d plugins from frontmatter (custom_token=%v)", len(plugins), pluginsToken != "") + pluginInfo := extractPluginsFromFrontmatter(result.Frontmatter) + if pluginInfo != nil && len(pluginInfo.Plugins) > 0 { + orchestratorToolsLog.Printf("Extracted %d plugins from frontmatter (custom_token=%v, mcp_configs=%d)", + len(pluginInfo.Plugins), pluginInfo.CustomToken != "", len(pluginInfo.MCPConfigs)) } // Merge plugins from imports with top-level plugins if len(importsResult.MergedPlugins) > 0 { + if pluginInfo == nil { + pluginInfo = &PluginInfo{ + MCPConfigs: make(map[string]*PluginMCPConfig), + } + } + orchestratorToolsLog.Printf("Merging %d plugins from imports", len(importsResult.MergedPlugins)) // Create a set to track unique plugins pluginsSet := make(map[string]bool) @@ -157,7 +163,7 @@ func (c *Compiler) processToolsAndMarkdown(result *parser.FrontmatterResult, cle } // Add top-level plugins (these override/supplement imports) - for _, plugin := range plugins { + for _, plugin := range pluginInfo.Plugins { pluginsSet[plugin] = true } @@ -169,9 +175,9 @@ func (c *Compiler) processToolsAndMarkdown(result *parser.FrontmatterResult, cle // Sort for deterministic output sort.Strings(mergedPlugins) - plugins = mergedPlugins + pluginInfo.Plugins = mergedPlugins - orchestratorToolsLog.Printf("Merged plugins: %d total unique plugins", len(plugins)) + orchestratorToolsLog.Printf("Merged plugins: %d total unique plugins", len(pluginInfo.Plugins)) } // Add MCP fetch server if needed (when web-fetch is requested but engine doesn't support it) @@ -292,8 +298,7 @@ func (c *Compiler) processToolsAndMarkdown(result *parser.FrontmatterResult, cle return &toolsProcessingResult{ tools: tools, runtimes: runtimes, - plugins: plugins, - pluginsToken: pluginsToken, + pluginInfo: pluginInfo, toolsTimeout: toolsTimeout, toolsStartupTimeout: toolsStartupTimeout, markdownContent: markdownContent, diff --git a/pkg/workflow/compiler_orchestrator_workflow.go b/pkg/workflow/compiler_orchestrator_workflow.go index 6fb846d995..57d9598ed8 100644 --- a/pkg/workflow/compiler_orchestrator_workflow.go +++ b/pkg/workflow/compiler_orchestrator_workflow.go @@ -122,8 +122,7 @@ func (c *Compiler) buildInitialWorkflowData( Tools: toolsResult.tools, ParsedTools: NewTools(toolsResult.tools), Runtimes: toolsResult.runtimes, - Plugins: toolsResult.plugins, - PluginsToken: toolsResult.pluginsToken, + PluginInfo: toolsResult.pluginInfo, MarkdownContent: toolsResult.markdownContent, AI: engineSetup.engineSetting, EngineConfig: engineSetup.engineConfig, diff --git a/pkg/workflow/compiler_types.go b/pkg/workflow/compiler_types.go index 43ea810a60..07a8b3759c 100644 --- a/pkg/workflow/compiler_types.go +++ b/pkg/workflow/compiler_types.go @@ -418,8 +418,7 @@ type WorkflowData struct { CacheMemoryConfig *CacheMemoryConfig // parsed cache-memory configuration RepoMemoryConfig *RepoMemoryConfig // parsed repo-memory configuration Runtimes map[string]any // runtime version overrides from frontmatter - Plugins []string // plugin repository slugs to install (e.g., ["org/repo", "org2/repo2"]) - PluginsToken string // custom github-token for plugin installation (from plugins.github-token field) + PluginInfo *PluginInfo // Consolidated plugin information (plugins, custom token, MCP configs) ToolsTimeout int // timeout in seconds for tool/MCP operations (0 = use engine default) GitHubToken string // top-level github-token expression from frontmatter ToolsStartupTimeout int // timeout in seconds for MCP server startup (0 = use engine default) diff --git a/pkg/workflow/copilot_engine_execution.go b/pkg/workflow/copilot_engine_execution.go index 531af7740f..ac2270e4fa 100644 --- a/pkg/workflow/copilot_engine_execution.go +++ b/pkg/workflow/copilot_engine_execution.go @@ -56,9 +56,9 @@ func (e *CopilotEngine) GetExecutionSteps(workflowData *WorkflowData, logFile st // Add Copilot config directory when plugins are declared so the CLI can discover installed plugins // Plugins are installed to ~/.copilot/plugins/ via copilot plugin install command // The CLI also reads plugin-index.json from ~/.copilot/ to discover installed plugins - if len(workflowData.Plugins) > 0 { + if workflowData.PluginInfo != nil && len(workflowData.PluginInfo.Plugins) > 0 { copilotArgs = append(copilotArgs, "--add-dir", "/home/runner/.copilot/") - copilotExecLog.Printf("Added Copilot config directory to --add-dir for plugin discovery (%d plugins)", len(workflowData.Plugins)) + copilotExecLog.Printf("Added Copilot config directory to --add-dir for plugin discovery (%d plugins)", len(workflowData.PluginInfo.Plugins)) } copilotExecLog.Print("Using firewall mode with simplified arguments") diff --git a/pkg/workflow/copilot_engine_installation.go b/pkg/workflow/copilot_engine_installation.go index daabc1669f..7656256571 100644 --- a/pkg/workflow/copilot_engine_installation.go +++ b/pkg/workflow/copilot_engine_installation.go @@ -143,14 +143,14 @@ func (e *CopilotEngine) GetInstallationSteps(workflowData *WorkflowData) []GitHu } // Add plugin installation steps after Copilot CLI installation - if len(workflowData.Plugins) > 0 { - copilotInstallLog.Printf("Adding plugin installation steps: %d plugins", len(workflowData.Plugins)) + if workflowData.PluginInfo != nil && len(workflowData.PluginInfo.Plugins) > 0 { + copilotInstallLog.Printf("Adding plugin installation steps: %d plugins", len(workflowData.PluginInfo.Plugins)) // Use plugin-specific token if provided, otherwise use top-level github-token - tokenToUse := workflowData.PluginsToken + tokenToUse := workflowData.PluginInfo.CustomToken if tokenToUse == "" { tokenToUse = workflowData.GitHubToken } - pluginSteps := GeneratePluginInstallationSteps(workflowData.Plugins, "copilot", tokenToUse) + pluginSteps := GeneratePluginInstallationSteps(workflowData.PluginInfo.Plugins, "copilot", tokenToUse) steps = append(steps, pluginSteps...) } diff --git a/pkg/workflow/copilot_engine_test.go b/pkg/workflow/copilot_engine_test.go index a16a1f0f7e..0ec25e98d6 100644 --- a/pkg/workflow/copilot_engine_test.go +++ b/pkg/workflow/copilot_engine_test.go @@ -1399,8 +1399,10 @@ func TestCopilotEnginePluginDiscoveryInSandboxMode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { workflowData := &WorkflowData{ - Name: "test-workflow", - Plugins: tt.plugins, + Name: "test-workflow", + PluginInfo: &PluginInfo{ + Plugins: tt.plugins, + }, NetworkPermissions: tt.networkPermissions, } steps := engine.GetExecutionSteps(workflowData, "/tmp/gh-aw/test.log") @@ -1446,8 +1448,10 @@ func TestCopilotEnginePluginDiscoveryWithSRT(t *testing.T) { // Test with SRT enabled (via sandbox config) workflowData := &WorkflowData{ - Name: "test-workflow", - Plugins: []string{"github/auto-agentics"}, + Name: "test-workflow", + PluginInfo: &PluginInfo{ + Plugins: []string{"github/auto-agentics"}, + }, SandboxConfig: &SandboxConfig{ Type: "sandbox-runtime", }, diff --git a/pkg/workflow/frontmatter_extraction_metadata.go b/pkg/workflow/frontmatter_extraction_metadata.go index 4d23f12ce3..46dfb369e7 100644 --- a/pkg/workflow/frontmatter_extraction_metadata.go +++ b/pkg/workflow/frontmatter_extraction_metadata.go @@ -256,36 +256,90 @@ func extractRuntimesFromFrontmatter(frontmatter map[string]any) map[string]any { } // extractPluginsFromFrontmatter extracts plugins configuration from frontmatter map -// Returns: (repos []string, customToken string) +// Returns: PluginInfo with plugins list, custom token, and per-plugin MCP configs // Supports both array format and object format with optional github-token -func extractPluginsFromFrontmatter(frontmatter map[string]any) ([]string, string) { +// Each plugin item can be either a string (repository slug) or an object with id and optional mcp config +func extractPluginsFromFrontmatter(frontmatter map[string]any) *PluginInfo { value, exists := frontmatter["plugins"] if !exists { - return nil, "" + return nil + } + + pluginInfo := &PluginInfo{ + MCPConfigs: make(map[string]*PluginMCPConfig), + } + + // Helper function to parse plugin items (can be string or object) + parsePluginItem := func(item any) (string, *PluginMCPConfig) { + // Try string format first: "org/repo" + if pluginStr, ok := item.(string); ok { + return pluginStr, nil + } + + // Try object format: { "id": "org/repo", "mcp": {...} } + if pluginObj, ok := item.(map[string]any); ok { + // Extract ID (required) + id, hasID := pluginObj["id"] + if !hasID { + return "", nil + } + idStr, ok := id.(string) + if !ok { + return "", nil + } + + // Extract MCP configuration (optional) + var mcpConfig *PluginMCPConfig + if mcpAny, hasMCP := pluginObj["mcp"]; hasMCP { + if mcpMap, ok := mcpAny.(map[string]any); ok { + mcpConfig = &PluginMCPConfig{} + + // Extract env variables + if envAny, hasEnv := mcpMap["env"]; hasEnv { + if envMap, ok := envAny.(map[string]any); ok { + mcpConfig.Env = make(map[string]string) + for k, v := range envMap { + if vStr, ok := v.(string); ok { + mcpConfig.Env[k] = vStr + } + } + } + } + } + } + + return idStr, mcpConfig + } + + return "", nil } - // Try array format first: ["org/repo1", "org/repo2"] + // Try array format first: ["org/repo1", { "id": "org/repo2", "mcp": {...} }] if pluginsArray, ok := value.([]any); ok { - var plugins []string for _, p := range pluginsArray { - if pluginStr, ok := p.(string); ok { - plugins = append(plugins, pluginStr) + id, mcpConfig := parsePluginItem(p) + if id != "" { + pluginInfo.Plugins = append(pluginInfo.Plugins, id) + if mcpConfig != nil { + pluginInfo.MCPConfigs[id] = mcpConfig + } } } - return plugins, "" + return pluginInfo } // Try object format: { "repos": [...], "github-token": "..." } if pluginsMap, ok := value.(map[string]any); ok { - var repos []string - var token string - - // Extract repos array + // Extract repos array (items can be strings or objects) if reposAny, hasRepos := pluginsMap["repos"]; hasRepos { if reposArray, ok := reposAny.([]any); ok { for _, r := range reposArray { - if repoStr, ok := r.(string); ok { - repos = append(repos, repoStr) + id, mcpConfig := parsePluginItem(r) + if id != "" { + pluginInfo.Plugins = append(pluginInfo.Plugins, id) + if mcpConfig != nil { + pluginInfo.MCPConfigs[id] = mcpConfig + } } } } @@ -294,12 +348,12 @@ func extractPluginsFromFrontmatter(frontmatter map[string]any) ([]string, string // Extract github-token (optional) if tokenAny, hasToken := pluginsMap["github-token"]; hasToken { if tokenStr, ok := tokenAny.(string); ok { - token = tokenStr + pluginInfo.CustomToken = tokenStr } } - return repos, token + return pluginInfo } - return nil, "" + return nil } diff --git a/pkg/workflow/frontmatter_types.go b/pkg/workflow/frontmatter_types.go index 92b34220e3..d2b1960727 100644 --- a/pkg/workflow/frontmatter_types.go +++ b/pkg/workflow/frontmatter_types.go @@ -49,8 +49,28 @@ type PermissionsConfig struct { OrganizationPackages string `json:"organization-packages,omitempty"` } -// PluginsConfig represents plugin configuration for installation -// Supports object format with repos list and optional custom github-token +// PluginMCPConfig represents MCP configuration for a plugin +type PluginMCPConfig struct { + Env map[string]string `json:"env,omitempty"` // Environment variables for MCP server instantiation +} + +// PluginItem represents configuration for a single plugin +// Supports both simple string format and object format with MCP configuration +type PluginItem struct { + ID string `json:"id"` // Plugin identifier/repository slug (e.g., "org/repo") + MCP *PluginMCPConfig `json:"mcp,omitempty"` // Optional MCP configuration +} + +// PluginInfo encapsulates all plugin-related configuration +// This consolidates plugins list, custom token, and per-plugin MCP configs +type PluginInfo struct { + Plugins []string // Plugin repository slugs to install + CustomToken string // Custom github-token for plugin installation + MCPConfigs map[string]*PluginMCPConfig // Per-plugin MCP configurations (keyed by plugin ID) +} + +// PluginsConfig represents plugin configuration for installation (for parsing only) +// Supports object format with repos list, optional custom github-token type PluginsConfig struct { Repos []string `json:"repos"` // List of plugin repository slugs (required) GitHubToken string `json:"github-token,omitempty"` // Custom GitHub token for plugin installation diff --git a/pkg/workflow/mcp_environment.go b/pkg/workflow/mcp_environment.go index 8146dc392a..bbd749fa87 100644 --- a/pkg/workflow/mcp_environment.go +++ b/pkg/workflow/mcp_environment.go @@ -208,5 +208,21 @@ func collectMCPEnvironmentVariables(tools map[string]any, mcpTools []string, wor } } + // Extract environment variables from plugin MCP configurations + // Plugins can define MCP servers with environment variables that need to be available during gateway setup + // We need to pass ALL env vars (not just secrets) since plugins may need configuration values + if workflowData != nil && workflowData.PluginInfo != nil && len(workflowData.PluginInfo.MCPConfigs) > 0 { + mcpEnvironmentLog.Printf("Extracting environment variables from %d plugin MCP configurations", len(workflowData.PluginInfo.MCPConfigs)) + for pluginID, mcpConfig := range workflowData.PluginInfo.MCPConfigs { + if mcpConfig != nil && len(mcpConfig.Env) > 0 { + mcpEnvironmentLog.Printf("Adding %d environment variables from plugin '%s' MCP configuration", len(mcpConfig.Env), pluginID) + // Add ALL environment variables from plugin MCP config (not just secrets) + for envVarName, envVarValue := range mcpConfig.Env { + envVars[envVarName] = envVarValue + } + } + } + } + return envVars } diff --git a/pkg/workflow/plugin_import_object_test.go b/pkg/workflow/plugin_import_object_test.go new file mode 100644 index 0000000000..93f8fe7933 --- /dev/null +++ b/pkg/workflow/plugin_import_object_test.go @@ -0,0 +1,100 @@ +//go:build !integration + +package workflow_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/github/gh-aw/pkg/workflow" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPluginImportWithObjectFormat(t *testing.T) { + tmpDir := t.TempDir() + + // Create shared plugins file with object format + sharedPlugins := `--- +plugins: + - github/imported-plugin1 + - id: github/imported-mcp-plugin + mcp: + env: + IMPORTED_KEY: ${{ secrets.IMPORTED_KEY }} +--- +` + sharedFile := filepath.Join(tmpDir, "shared-plugins.md") + err := os.WriteFile(sharedFile, []byte(sharedPlugins), 0644) + require.NoError(t, err, "Failed to write shared plugins file") + + // Create main workflow that imports the shared plugins + mainWorkflow := `--- +engine: copilot +on: workflow_dispatch +permissions: + issues: read + pull-requests: read + contents: read +imports: + - shared-plugins.md +plugins: + - github/top-level-plugin + - id: github/top-level-mcp-plugin + mcp: + env: + TOP_KEY: ${{ secrets.TOP_KEY }} +--- + +Test plugin imports with object format +` + mainFile := filepath.Join(tmpDir, "test-workflow.md") + err = os.WriteFile(mainFile, []byte(mainWorkflow), 0644) + require.NoError(t, err, "Failed to write main workflow file") + + // Compile workflow + compiler := workflow.NewCompiler() + err = compiler.CompileWorkflow(mainFile) + require.NoError(t, err, "Compilation should succeed") + + // Read generated lock file + lockFile := strings.Replace(mainFile, ".md", ".lock.yml", 1) + content, err := os.ReadFile(lockFile) + require.NoError(t, err, "Failed to read lock file") + + lockContent := string(content) + + // Verify all plugins are installed (imported + top-level) + assert.Contains(t, lockContent, "copilot plugin install github/imported-plugin1", + "Should install imported plugin1") + assert.Contains(t, lockContent, "copilot plugin install github/imported-mcp-plugin", + "Should install imported MCP plugin") + assert.Contains(t, lockContent, "copilot plugin install github/top-level-plugin", + "Should install top-level plugin") + assert.Contains(t, lockContent, "copilot plugin install github/top-level-mcp-plugin", + "Should install top-level MCP plugin") + + // Verify MCP environment variables are in the gateway step + assert.Contains(t, lockContent, "Start MCP gateway", + "Should have MCP gateway step") + + // Extract the MCP gateway section + startIdx := strings.Index(lockContent, "Start MCP gateway") + require.Positive(t, startIdx, "MCP gateway step should exist") + + endIdx := strings.Index(lockContent[startIdx:], "- name:") + if endIdx == -1 { + endIdx = len(lockContent) + } else { + endIdx = startIdx + endIdx + } + mcpGatewaySection := lockContent[startIdx:endIdx] + + // Verify top-level MCP env vars are present + // (imported plugin MCP configs would be defined in the shared file's own execution, + // not merged into the main workflow's MCP gateway) + assert.Contains(t, mcpGatewaySection, "TOP_KEY: ${{ secrets.TOP_KEY }}", + "Should contain top-level MCP environment variable") +} diff --git a/pkg/workflow/plugin_installation_test.go b/pkg/workflow/plugin_installation_test.go index a30c4678bb..bb688e671d 100644 --- a/pkg/workflow/plugin_installation_test.go +++ b/pkg/workflow/plugin_installation_test.go @@ -167,7 +167,13 @@ func TestExtractPluginsFromFrontmatter(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - repos, token := extractPluginsFromFrontmatter(tt.frontmatter) + pluginInfo := extractPluginsFromFrontmatter(tt.frontmatter) + var repos []string + var token string + if pluginInfo != nil { + repos = pluginInfo.Plugins + token = pluginInfo.CustomToken + } assert.Equal(t, tt.expectedRepos, repos, "Extracted plugin repos should match expected") assert.Equal(t, tt.expectedToken, token, "Extracted plugin token should match expected") }) @@ -189,8 +195,10 @@ func TestPluginInstallationIntegration(t *testing.T) { t.Run(e.engineID, func(t *testing.T) { // Create workflow data with plugins workflowData := &WorkflowData{ - Name: "test-workflow", - Plugins: []string{"github/test-plugin"}, + Name: "test-workflow", + PluginInfo: &PluginInfo{ + Plugins: []string{"github/test-plugin"}, + }, } // Get installation steps @@ -263,9 +271,11 @@ func TestPluginObjectFormatWithCustomToken(t *testing.T) { t.Run(e.engineID, func(t *testing.T) { // Create workflow data with plugins and custom token workflowData := &WorkflowData{ - Name: "test-workflow", - Plugins: []string{"github/test-plugin"}, - PluginsToken: "${{ secrets.CUSTOM_PLUGIN_TOKEN }}", + Name: "test-workflow", + PluginInfo: &PluginInfo{ + Plugins: []string{"github/test-plugin"}, + CustomToken: "${{ secrets.CUSTOM_PLUGIN_TOKEN }}", + }, } // Get installation steps diff --git a/pkg/workflow/plugin_mcp_integration_test.go b/pkg/workflow/plugin_mcp_integration_test.go new file mode 100644 index 0000000000..69be282883 --- /dev/null +++ b/pkg/workflow/plugin_mcp_integration_test.go @@ -0,0 +1,224 @@ +//go:build integration + +package workflow + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/github/gh-aw/pkg/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPluginMCPCompilation(t *testing.T) { + tmpDir := testutil.TempDir(t, "plugin-mcp-test") + + tests := []struct { + name string + workflow string + expectedPlugins []string + expectedEnvVars map[string]string + shouldNotContain []string + }{ + { + name: "Plugin with MCP env configuration", + workflow: `--- +engine: copilot +on: workflow_dispatch +permissions: + issues: read + pull-requests: read + contents: read +plugins: + - github/simple-plugin + - id: github/mcp-plugin + mcp: + env: + API_KEY: ${{ secrets.API_KEY }} + API_URL: https://api.example.com +--- + +Test plugin with MCP configuration +`, + expectedPlugins: []string{ + "copilot plugin install github/simple-plugin", + "copilot plugin install github/mcp-plugin", + }, + expectedEnvVars: map[string]string{ + "API_KEY": "${{ secrets.API_KEY }}", + "API_URL": "https://api.example.com", + }, + shouldNotContain: []string{}, + }, + { + name: "Multiple plugins with different MCP configs", + workflow: `--- +engine: claude +on: workflow_dispatch +permissions: + issues: read + pull-requests: read + contents: read +plugins: + - id: github/plugin1 + mcp: + env: + PLUGIN1_KEY: ${{ secrets.PLUGIN1_KEY }} + - id: github/plugin2 + mcp: + env: + PLUGIN2_KEY: ${{ secrets.PLUGIN2_KEY }} + PLUGIN2_URL: https://plugin2.example.com +--- + +Test multiple plugins with MCP configs +`, + expectedPlugins: []string{ + "claude plugin install github/plugin1", + "claude plugin install github/plugin2", + }, + expectedEnvVars: map[string]string{ + "PLUGIN1_KEY": "${{ secrets.PLUGIN1_KEY }}", + "PLUGIN2_KEY": "${{ secrets.PLUGIN2_KEY }}", + "PLUGIN2_URL": "https://plugin2.example.com", + }, + shouldNotContain: []string{}, + }, + { + name: "Mixed simple and MCP-enabled plugins", + workflow: `--- +engine: codex +on: workflow_dispatch +permissions: + issues: read + pull-requests: read + contents: read +plugins: + repos: + - github/simple1 + - id: github/mcp-enabled + mcp: + env: + MCP_SECRET: ${{ secrets.MCP_SECRET }} + - github/simple2 + github-token: ${{ secrets.CUSTOM_TOKEN }} +--- + +Test mixed plugin configuration +`, + expectedPlugins: []string{ + "codex plugin install github/simple1", + "codex plugin install github/mcp-enabled", + "codex plugin install github/simple2", + }, + expectedEnvVars: map[string]string{ + "MCP_SECRET": "${{ secrets.MCP_SECRET }}", + }, + shouldNotContain: []string{ + "GH_AW_PLUGINS_TOKEN", // Should use custom token instead + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create test file + testFile := filepath.Join(tmpDir, "test-plugin-mcp.md") + err := os.WriteFile(testFile, []byte(tt.workflow), 0644) + require.NoError(t, err, "Failed to write test file") + + // Compile workflow + compiler := NewCompiler() + err = compiler.CompileWorkflow(testFile) + require.NoError(t, err, "Compilation should succeed") + + // Read generated lock file + lockFile := strings.Replace(testFile, ".md", ".lock.yml", 1) + content, err := os.ReadFile(lockFile) + require.NoError(t, err, "Failed to read lock file") + + lockContent := string(content) + + // Verify all expected plugin install commands are present + for _, expectedPlugin := range tt.expectedPlugins { + assert.Contains(t, lockContent, expectedPlugin, + "Lock file should contain plugin install command: %s", expectedPlugin) + } + + // Verify MCP gateway step exists + assert.Contains(t, lockContent, "Start MCP gateway", + "Lock file should contain MCP gateway step") + + // Extract the MCP gateway step section + startIdx := strings.Index(lockContent, "Start MCP gateway") + require.Greater(t, startIdx, 0, "MCP gateway step should exist") + + // Find the end of the MCP gateway step (next step or end of job) + endIdx := strings.Index(lockContent[startIdx:], "- name:") + if endIdx == -1 { + endIdx = len(lockContent) + } else { + endIdx = startIdx + endIdx + } + mcpGatewaySection := lockContent[startIdx:endIdx] + + // Verify all expected environment variables are in the MCP gateway step + for envVar, expectedValue := range tt.expectedEnvVars { + expectedLine := envVar + ": " + expectedValue + assert.Contains(t, mcpGatewaySection, expectedLine, + "MCP gateway step should contain environment variable: %s", expectedLine) + } + + // Verify items that should NOT be present + for _, shouldNotContainStr := range tt.shouldNotContain { + assert.NotContains(t, lockContent, shouldNotContainStr, + "Lock file should not contain: %s", shouldNotContainStr) + } + }) + } +} + +func TestPluginMCPBackwardCompatibility(t *testing.T) { + tmpDir := testutil.TempDir(t, "plugin-backward-compat-test") + + // Test that existing plugin formats still work + workflow := `--- +engine: copilot +on: workflow_dispatch +permissions: + issues: read + pull-requests: read + contents: read +plugins: + - github/plugin1 + - github/plugin2 +--- + +Test backward compatibility +` + + testFile := filepath.Join(tmpDir, "test-backward-compat.md") + err := os.WriteFile(testFile, []byte(workflow), 0644) + require.NoError(t, err, "Failed to write test file") + + // Compile workflow + compiler := NewCompiler() + err = compiler.CompileWorkflow(testFile) + require.NoError(t, err, "Compilation should succeed for backward compatible format") + + // Read generated lock file + lockFile := strings.Replace(testFile, ".md", ".lock.yml", 1) + content, err := os.ReadFile(lockFile) + require.NoError(t, err, "Failed to read lock file") + + lockContent := string(content) + + // Verify plugins are installed + assert.Contains(t, lockContent, "copilot plugin install github/plugin1", + "Should install plugin1") + assert.Contains(t, lockContent, "copilot plugin install github/plugin2", + "Should install plugin2") +} diff --git a/pkg/workflow/plugin_mcp_test.go b/pkg/workflow/plugin_mcp_test.go new file mode 100644 index 0000000000..3ced95d2b6 --- /dev/null +++ b/pkg/workflow/plugin_mcp_test.go @@ -0,0 +1,178 @@ +//go:build !integration + +package workflow + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExtractPluginsFromFrontmatter_WithMCPConfig(t *testing.T) { + tests := []struct { + name string + frontmatter map[string]any + expectedRepos []string + expectedToken string + expectedMCPConfigs map[string]*PluginMCPConfig + }{ + { + name: "Array format with simple string plugin", + frontmatter: map[string]any{ + "plugins": []any{"github/plugin1", "github/plugin2"}, + }, + expectedRepos: []string{"github/plugin1", "github/plugin2"}, + expectedToken: "", + expectedMCPConfigs: map[string]*PluginMCPConfig{}, + }, + { + name: "Array format with plugin object containing MCP config", + frontmatter: map[string]any{ + "plugins": []any{ + "github/simple-plugin", + map[string]any{ + "id": "github/mcp-plugin", + "mcp": map[string]any{ + "env": map[string]any{ + "API_KEY": "${{ secrets.API_KEY }}", + "API_URL": "https://api.example.com", + }, + }, + }, + }, + }, + expectedRepos: []string{"github/simple-plugin", "github/mcp-plugin"}, + expectedToken: "", + expectedMCPConfigs: map[string]*PluginMCPConfig{ + "github/mcp-plugin": { + Env: map[string]string{ + "API_KEY": "${{ secrets.API_KEY }}", + "API_URL": "https://api.example.com", + }, + }, + }, + }, + { + name: "Object format with custom token and mixed plugin types", + frontmatter: map[string]any{ + "plugins": map[string]any{ + "repos": []any{ + "github/simple-plugin", + map[string]any{ + "id": "github/mcp-plugin", + "mcp": map[string]any{ + "env": map[string]any{ + "SECRET_KEY": "${{ secrets.SECRET_KEY }}", + }, + }, + }, + }, + "github-token": "${{ secrets.CUSTOM_TOKEN }}", + }, + }, + expectedRepos: []string{"github/simple-plugin", "github/mcp-plugin"}, + expectedToken: "${{ secrets.CUSTOM_TOKEN }}", + expectedMCPConfigs: map[string]*PluginMCPConfig{ + "github/mcp-plugin": { + Env: map[string]string{ + "SECRET_KEY": "${{ secrets.SECRET_KEY }}", + }, + }, + }, + }, + { + name: "Multiple plugins with different MCP configs", + frontmatter: map[string]any{ + "plugins": []any{ + map[string]any{ + "id": "github/plugin1", + "mcp": map[string]any{ + "env": map[string]any{ + "API_KEY_1": "${{ secrets.API_KEY_1 }}", + }, + }, + }, + map[string]any{ + "id": "github/plugin2", + "mcp": map[string]any{ + "env": map[string]any{ + "API_KEY_2": "${{ secrets.API_KEY_2 }}", + }, + }, + }, + }, + }, + expectedRepos: []string{"github/plugin1", "github/plugin2"}, + expectedToken: "", + expectedMCPConfigs: map[string]*PluginMCPConfig{ + "github/plugin1": { + Env: map[string]string{ + "API_KEY_1": "${{ secrets.API_KEY_1 }}", + }, + }, + "github/plugin2": { + Env: map[string]string{ + "API_KEY_2": "${{ secrets.API_KEY_2 }}", + }, + }, + }, + }, + { + name: "Plugin object with URL but no MCP config", + frontmatter: map[string]any{ + "plugins": []any{ + map[string]any{ + "id": "github/simple-plugin", + }, + }, + }, + expectedRepos: []string{"github/simple-plugin"}, + expectedToken: "", + expectedMCPConfigs: map[string]*PluginMCPConfig{}, + }, + { + name: "No plugins defined", + frontmatter: map[string]any{}, + expectedRepos: nil, + expectedToken: "", + expectedMCPConfigs: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pluginInfo := extractPluginsFromFrontmatter(tt.frontmatter) + + var repos []string + var token string + var mcpConfigs map[string]*PluginMCPConfig + + if pluginInfo != nil { + repos = pluginInfo.Plugins + token = pluginInfo.CustomToken + mcpConfigs = pluginInfo.MCPConfigs + } + + assert.Equal(t, tt.expectedRepos, repos, "Extracted plugin repos should match expected") + assert.Equal(t, tt.expectedToken, token, "Extracted plugin token should match expected") + + if tt.expectedMCPConfigs == nil { + if mcpConfigs != nil { + assert.Empty(t, mcpConfigs, "MCP configs should be empty when none expected") + } + } else { + require.NotNil(t, mcpConfigs, "MCP configs should not be nil") + assert.Len(t, mcpConfigs, len(tt.expectedMCPConfigs), "Number of MCP configs should match") + + for id, expectedConfig := range tt.expectedMCPConfigs { + actualConfig, exists := mcpConfigs[id] + assert.True(t, exists, "MCP config for %s should exist", id) + if exists { + assert.Equal(t, expectedConfig.Env, actualConfig.Env, "Env vars for %s should match", id) + } + } + } + }) + } +}