diff --git a/pkg/parser/content_extractor.go b/pkg/parser/content_extractor.go index 7fc78852d8..bbac920192 100644 --- a/pkg/parser/content_extractor.go +++ b/pkg/parser/content_extractor.go @@ -145,6 +145,11 @@ func extractBotsFromContent(content string) (string, error) { return extractFrontmatterField(content, "bots", "[]") } +// extractPluginsFromContent extracts plugins section from frontmatter as JSON string +func extractPluginsFromContent(content string) (string, error) { + return extractFrontmatterField(content, "plugins", "[]") +} + // extractPostStepsFromContent extracts post-steps section from frontmatter as YAML string func extractPostStepsFromContent(content string) (string, error) { result, err := ExtractFrontmatterFromContent(content) diff --git a/pkg/parser/import_processor.go b/pkg/parser/import_processor.go index d9d8dcc94f..3dd650d446 100644 --- a/pkg/parser/import_processor.go +++ b/pkg/parser/import_processor.go @@ -29,6 +29,7 @@ type ImportsResult struct { MergedPermissions string // Merged permissions configuration from all imports MergedSecretMasking string // Merged secret-masking steps from all imports MergedBots []string // Merged bots list from all imports (union of bot names) + MergedPlugins []string // Merged plugins list from all imports (union of plugin repos) MergedPostSteps string // Merged post-steps configuration from all imports (appended in order) MergedLabels []string // Merged labels from all imports (union of label names) MergedCaches []string // Merged cache configurations from all imports (appended in order) @@ -180,6 +181,8 @@ func processImportsFromFrontmatterWithManifestAndSource(frontmatter map[string]a var safeInputs []string var bots []string // Track unique bot names botsSet := make(map[string]bool) // Set for deduplicating bots + var plugins []string // Track unique plugin repos + pluginsSet := make(map[string]bool) // Set for deduplicating plugins var labels []string // Track unique labels labelsSet := make(map[string]bool) // Set for deduplicating labels var caches []string // Track cache configurations (appended in order) @@ -544,6 +547,21 @@ func processImportsFromFrontmatterWithManifestAndSource(frontmatter map[string]a } } + // Extract plugins from imported file (merge into set to avoid duplicates) + pluginsContent, err := extractPluginsFromContent(string(content)) + if err == nil && pluginsContent != "" && pluginsContent != "[]" { + // Parse plugins JSON array + var importedPlugins []string + if jsonErr := json.Unmarshal([]byte(pluginsContent), &importedPlugins); jsonErr == nil { + for _, plugin := range importedPlugins { + if !pluginsSet[plugin] { + pluginsSet[plugin] = true + plugins = append(plugins, plugin) + } + } + } + } + // Extract post-steps from imported file (append in order) postStepsContent, err := extractPostStepsFromContent(string(content)) if err == nil && postStepsContent != "" { @@ -593,6 +611,7 @@ func processImportsFromFrontmatterWithManifestAndSource(frontmatter map[string]a MergedPermissions: permissionsBuilder.String(), MergedSecretMasking: secretMaskingBuilder.String(), MergedBots: bots, + MergedPlugins: plugins, MergedPostSteps: postStepsBuilder.String(), MergedLabels: labels, MergedCaches: caches, diff --git a/pkg/workflow/compiler_orchestrator_tools.go b/pkg/workflow/compiler_orchestrator_tools.go index 8e7086c332..da97155d20 100644 --- a/pkg/workflow/compiler_orchestrator_tools.go +++ b/pkg/workflow/compiler_orchestrator_tools.go @@ -144,6 +144,35 @@ func (c *Compiler) processToolsAndMarkdown(result *parser.FrontmatterResult, cle orchestratorToolsLog.Printf("Extracted %d plugins from frontmatter (custom_token=%v)", len(plugins), pluginsToken != "") } + // Merge plugins from imports with top-level plugins + if len(importsResult.MergedPlugins) > 0 { + orchestratorToolsLog.Printf("Merging %d plugins from imports", len(importsResult.MergedPlugins)) + // Create a set to track unique plugins + pluginsSet := make(map[string]bool) + + // Add imported plugins first (imports have lower priority) + for _, plugin := range importsResult.MergedPlugins { + pluginsSet[plugin] = true + } + + // Add top-level plugins (these override/supplement imports) + for _, plugin := range plugins { + pluginsSet[plugin] = true + } + + // Convert set back to slice + mergedPlugins := make([]string, 0, len(pluginsSet)) + for plugin := range pluginsSet { + mergedPlugins = append(mergedPlugins, plugin) + } + + // Sort for deterministic output + sort.Strings(mergedPlugins) + plugins = mergedPlugins + + orchestratorToolsLog.Printf("Merged plugins: %d total unique plugins", len(plugins)) + } + // Add MCP fetch server if needed (when web-fetch is requested but engine doesn't support it) tools, _ = AddMCPFetchServerIfNeeded(tools, agenticEngine) diff --git a/pkg/workflow/plugin_import_test.go b/pkg/workflow/plugin_import_test.go new file mode 100644 index 0000000000..214d854677 --- /dev/null +++ b/pkg/workflow/plugin_import_test.go @@ -0,0 +1,309 @@ +//go:build !integration + +package workflow_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/github/gh-aw/pkg/stringutil" + "github.com/github/gh-aw/pkg/testutil" + "github.com/github/gh-aw/pkg/workflow" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCompileWorkflowWithPluginImports(t *testing.T) { + // Create a temporary directory for test files + tempDir := testutil.TempDir(t, "test-*") + + // Create a shared plugins file + sharedPluginsPath := filepath.Join(tempDir, "shared-plugins.md") + sharedPluginsContent := `--- +on: push +plugins: + - github/plugin-one + - github/plugin-two +--- +` + require.NoError(t, os.WriteFile(sharedPluginsPath, []byte(sharedPluginsContent), 0644), + "Failed to write shared plugins file") + + // Create a workflow file that imports the shared plugins + workflowPath := filepath.Join(tempDir, "test-workflow.md") + workflowContent := `--- +on: issues +engine: copilot +imports: + - shared-plugins.md +--- + +# Test Workflow + +This is a test workflow with imported plugins. +` + require.NoError(t, os.WriteFile(workflowPath, []byte(workflowContent), 0644), + "Failed to write workflow file") + + // Compile the workflow + compiler := workflow.NewCompiler() + require.NoError(t, compiler.CompileWorkflow(workflowPath), + "CompileWorkflow should succeed") + + // Read the generated lock file + lockFilePath := stringutil.MarkdownToLockFile(workflowPath) + lockFileContent, err := os.ReadFile(lockFilePath) + require.NoError(t, err, "Failed to read lock file") + + workflowData := string(lockFileContent) + + // Verify that the compiled workflow contains the imported plugins + assert.Contains(t, workflowData, "copilot plugin install github/plugin-one", + "Expected workflow to install plugin-one from import") + assert.Contains(t, workflowData, "copilot plugin install github/plugin-two", + "Expected workflow to install plugin-two from import") +} + +func TestCompileWorkflowWithPluginImportsAndTopLevelPlugins(t *testing.T) { + // Create a temporary directory for test files + tempDir := testutil.TempDir(t, "test-*") + + // Create a shared plugins file + sharedPluginsPath := filepath.Join(tempDir, "shared-plugins.md") + sharedPluginsContent := `--- +on: push +plugins: + - github/imported-plugin +--- +` + require.NoError(t, os.WriteFile(sharedPluginsPath, []byte(sharedPluginsContent), 0644), + "Failed to write shared plugins file") + + // Create a workflow file that imports plugins and defines its own + workflowPath := filepath.Join(tempDir, "test-workflow.md") + workflowContent := `--- +on: issues +engine: copilot +imports: + - shared-plugins.md +plugins: + - github/top-level-plugin +--- + +# Test Workflow + +This workflow has both imported and top-level plugins. +` + require.NoError(t, os.WriteFile(workflowPath, []byte(workflowContent), 0644), + "Failed to write workflow file") + + // Compile the workflow + compiler := workflow.NewCompiler() + require.NoError(t, compiler.CompileWorkflow(workflowPath), + "CompileWorkflow should succeed") + + // Read the generated lock file + lockFilePath := stringutil.MarkdownToLockFile(workflowPath) + lockFileContent, err := os.ReadFile(lockFilePath) + require.NoError(t, err, "Failed to read lock file") + + workflowData := string(lockFileContent) + + // Verify that both imported and top-level plugins are included + assert.Contains(t, workflowData, "copilot plugin install github/imported-plugin", + "Expected workflow to install imported-plugin from import") + assert.Contains(t, workflowData, "copilot plugin install github/top-level-plugin", + "Expected workflow to install top-level-plugin from main workflow") +} + +func TestCompileWorkflowWithMultiplePluginImports(t *testing.T) { + // Create a temporary directory for test files + tempDir := testutil.TempDir(t, "test-*") + + // Create first shared plugins file + sharedPlugins1Path := filepath.Join(tempDir, "plugins-1.md") + sharedPlugins1Content := `--- +on: push +plugins: + - github/plugin-a + - github/plugin-b +--- +` + require.NoError(t, os.WriteFile(sharedPlugins1Path, []byte(sharedPlugins1Content), 0644), + "Failed to write first plugins file") + + // Create second shared plugins file + sharedPlugins2Path := filepath.Join(tempDir, "plugins-2.md") + sharedPlugins2Content := `--- +on: push +plugins: + - github/plugin-c +--- +` + require.NoError(t, os.WriteFile(sharedPlugins2Path, []byte(sharedPlugins2Content), 0644), + "Failed to write second plugins file") + + // Create a workflow file that imports both plugin files + workflowPath := filepath.Join(tempDir, "test-workflow.md") + workflowContent := `--- +on: issues +engine: copilot +imports: + - plugins-1.md + - plugins-2.md +--- + +# Test Workflow + +This workflow imports plugins from multiple files. +` + require.NoError(t, os.WriteFile(workflowPath, []byte(workflowContent), 0644), + "Failed to write workflow file") + + // Compile the workflow + compiler := workflow.NewCompiler() + require.NoError(t, compiler.CompileWorkflow(workflowPath), + "CompileWorkflow should succeed") + + // Read the generated lock file + lockFilePath := stringutil.MarkdownToLockFile(workflowPath) + lockFileContent, err := os.ReadFile(lockFilePath) + require.NoError(t, err, "Failed to read lock file") + + workflowData := string(lockFileContent) + + // Verify all plugins from both imports are included + assert.Contains(t, workflowData, "copilot plugin install github/plugin-a", + "Expected workflow to install plugin-a") + assert.Contains(t, workflowData, "copilot plugin install github/plugin-b", + "Expected workflow to install plugin-b") + assert.Contains(t, workflowData, "copilot plugin install github/plugin-c", + "Expected workflow to install plugin-c") +} + +func TestCompileWorkflowWithDuplicatePluginImports(t *testing.T) { + // Create a temporary directory for test files + tempDir := testutil.TempDir(t, "test-*") + + // Create first shared plugins file with duplicate plugin + sharedPlugins1Path := filepath.Join(tempDir, "plugins-1.md") + sharedPlugins1Content := `--- +on: push +plugins: + - github/shared-plugin + - github/plugin-a +--- +` + require.NoError(t, os.WriteFile(sharedPlugins1Path, []byte(sharedPlugins1Content), 0644), + "Failed to write first plugins file") + + // Create second shared plugins file with the same shared plugin + sharedPlugins2Path := filepath.Join(tempDir, "plugins-2.md") + sharedPlugins2Content := `--- +on: push +plugins: + - github/shared-plugin + - github/plugin-b +--- +` + require.NoError(t, os.WriteFile(sharedPlugins2Path, []byte(sharedPlugins2Content), 0644), + "Failed to write second plugins file") + + // Create a workflow file that imports both files and also defines the duplicate plugin + workflowPath := filepath.Join(tempDir, "test-workflow.md") + workflowContent := `--- +on: issues +engine: copilot +imports: + - plugins-1.md + - plugins-2.md +plugins: + - github/shared-plugin + - github/top-level-plugin +--- + +# Test Workflow + +This workflow has duplicate plugins across imports and top-level. +` + require.NoError(t, os.WriteFile(workflowPath, []byte(workflowContent), 0644), + "Failed to write workflow file") + + // Compile the workflow + compiler := workflow.NewCompiler() + require.NoError(t, compiler.CompileWorkflow(workflowPath), + "CompileWorkflow should succeed") + + // Read the generated lock file + lockFilePath := stringutil.MarkdownToLockFile(workflowPath) + lockFileContent, err := os.ReadFile(lockFilePath) + require.NoError(t, err, "Failed to read lock file") + + workflowData := string(lockFileContent) + + // Count occurrences of the shared plugin installation + installCmd := "copilot plugin install github/shared-plugin" + count := strings.Count(workflowData, installCmd) + + // Verify the shared plugin is only installed once (deduplicated) + assert.Equal(t, 1, count, + "Expected shared-plugin to be installed only once despite appearing in multiple imports") + + // Verify all unique plugins are included + assert.Contains(t, workflowData, "copilot plugin install github/plugin-a", + "Expected workflow to install plugin-a") + assert.Contains(t, workflowData, "copilot plugin install github/plugin-b", + "Expected workflow to install plugin-b") + assert.Contains(t, workflowData, "copilot plugin install github/top-level-plugin", + "Expected workflow to install top-level-plugin") +} + +func TestCompileWorkflowWithPluginImportsClaudeEngine(t *testing.T) { + // Create a temporary directory for test files + tempDir := testutil.TempDir(t, "test-*") + + // Create a shared plugins file + sharedPluginsPath := filepath.Join(tempDir, "shared-plugins.md") + sharedPluginsContent := `--- +on: push +plugins: + - anthropic/plugin-one +--- +` + require.NoError(t, os.WriteFile(sharedPluginsPath, []byte(sharedPluginsContent), 0644), + "Failed to write shared plugins file") + + // Create a workflow file that imports plugins and uses Claude engine + workflowPath := filepath.Join(tempDir, "test-workflow.md") + workflowContent := `--- +on: issues +engine: claude +imports: + - shared-plugins.md +--- + +# Test Workflow + +This workflow uses Claude engine with imported plugins. +` + require.NoError(t, os.WriteFile(workflowPath, []byte(workflowContent), 0644), + "Failed to write workflow file") + + // Compile the workflow + compiler := workflow.NewCompiler() + require.NoError(t, compiler.CompileWorkflow(workflowPath), + "CompileWorkflow should succeed") + + // Read the generated lock file + lockFilePath := stringutil.MarkdownToLockFile(workflowPath) + lockFileContent, err := os.ReadFile(lockFilePath) + require.NoError(t, err, "Failed to read lock file") + + workflowData := string(lockFileContent) + + // Verify that Claude engine installs the plugin + assert.Contains(t, workflowData, "claude plugin install anthropic/plugin-one", + "Expected Claude engine to install plugin from import") +}