diff --git a/.github/workflows/glossary-maintainer.lock.yml b/.github/workflows/glossary-maintainer.lock.yml index dfb9644617..a420e471f2 100644 --- a/.github/workflows/glossary-maintainer.lock.yml +++ b/.github/workflows/glossary-maintainer.lock.yml @@ -111,6 +111,18 @@ jobs: uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 with: persist-credentials: false + - name: Merge remote .github folder + if: ${{ always() }} + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + GH_AW_AGENT_FILE: ".github/agents/technical-doc-writer.agent.md" + GH_AW_AGENT_IMPORT_SPEC: "../agents/technical-doc-writer.agent.md" + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/merge_remote_agent_github_folder.cjs'); + await main(); - name: Create gh-aw temp directory run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh # Cache memory file share configuration from frontmatter processed below diff --git a/.github/workflows/hourly-ci-cleaner.lock.yml b/.github/workflows/hourly-ci-cleaner.lock.yml index 04e3d53918..1de1e0a6a6 100644 --- a/.github/workflows/hourly-ci-cleaner.lock.yml +++ b/.github/workflows/hourly-ci-cleaner.lock.yml @@ -113,6 +113,18 @@ jobs: uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 with: persist-credentials: false + - name: Merge remote .github folder + if: ${{ always() }} + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + GH_AW_AGENT_FILE: ".github/agents/ci-cleaner.agent.md" + GH_AW_AGENT_IMPORT_SPEC: "../agents/ci-cleaner.agent.md" + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/merge_remote_agent_github_folder.cjs'); + await main(); - name: Setup Node.js uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0 with: diff --git a/.github/workflows/scout.lock.yml b/.github/workflows/scout.lock.yml index f8d6aab6f9..87aaf0eb93 100644 --- a/.github/workflows/scout.lock.yml +++ b/.github/workflows/scout.lock.yml @@ -186,6 +186,17 @@ jobs: uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 with: persist-credentials: false + - name: Merge remote .github folder + if: ${{ always() }} + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + GH_AW_REPOSITORY_IMPORTS: '["github/github-deep-research-agent@main"]' + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/merge_remote_agent_github_folder.cjs'); + await main(); - name: Create gh-aw temp directory run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh - name: Setup jq utilities directory diff --git a/.github/workflows/scout.md b/.github/workflows/scout.md index ed1bd9d559..92e76a875b 100644 --- a/.github/workflows/scout.md +++ b/.github/workflows/scout.md @@ -16,6 +16,7 @@ permissions: roles: [admin, maintainer, write] engine: claude imports: + - github/github-deep-research-agent@main - shared/reporting.md - shared/mcp/arxiv.md - shared/mcp/tavily.md @@ -40,7 +41,7 @@ strict: true # Scout Deep Research Agent -You are the Scout agent - an expert research assistant that performs deep, comprehensive investigations using web search capabilities. +You are the Scout agent - an expert research assistant that performs deep, comprehensive investigations using web search capabilities and the imported GitHub deep research agent tools. ## Mission @@ -61,6 +62,8 @@ When invoked with the `/scout` command in an issue or pull request comment, OR m **Note**: If a research topic is provided above (from workflow_dispatch), use that as your primary research focus. Otherwise, analyze the triggering content to determine the research topic. +**Deep Research Agent**: This workflow imports the GitHub deep research agent repository, which provides additional tools and capabilities from `.github/agents/` and `.github/workflows/` for enhanced research functionality. + ## Research Process ### 1. Context Analysis diff --git a/.github/workflows/technical-doc-writer.lock.yml b/.github/workflows/technical-doc-writer.lock.yml index 8c1ad461ab..e7c14d6185 100644 --- a/.github/workflows/technical-doc-writer.lock.yml +++ b/.github/workflows/technical-doc-writer.lock.yml @@ -114,6 +114,18 @@ jobs: uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 with: persist-credentials: false + - name: Merge remote .github folder + if: ${{ always() }} + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + GH_AW_AGENT_FILE: ".github/agents/technical-doc-writer.agent.md" + GH_AW_AGENT_IMPORT_SPEC: "../agents/technical-doc-writer.agent.md" + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/merge_remote_agent_github_folder.cjs'); + await main(); - name: Setup Node.js uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0 with: diff --git a/actions/setup/js/merge_remote_agent_github_folder.cjs b/actions/setup/js/merge_remote_agent_github_folder.cjs new file mode 100644 index 0000000000..e8dbb903b4 --- /dev/null +++ b/actions/setup/js/merge_remote_agent_github_folder.cjs @@ -0,0 +1,380 @@ +// @ts-check +/// + +/** + * Merge remote repository's .github folder into current repository + * + * This script handles importing .github folder content from remote repositories. + * It uses sparse checkout to efficiently download only the .github folder + * and merges it into the current repository, failing on conflicts. + * + * This script runs in a github-script context where core object + * is available globally. Do NOT use npm packages from the actions org. + * + * Environment Variables: + * - GH_AW_REPOSITORY_IMPORTS: JSON array of repository imports (e.g., '["owner/repo@ref"]') + * - GH_AW_AGENT_FILE: Path to the agent file (e.g., ".github/agents/my-agent.md") [legacy] + * - GH_AW_AGENT_IMPORT_SPEC: Import specification (e.g., "owner/repo/.github/agents/agent.md@v1.0.0") [legacy] + * - GITHUB_WORKSPACE: Path to the current repository workspace + */ + +const fs = require("fs"); +const path = require("path"); +const { execSync } = require("child_process"); + +const { getErrorMessage } = require("./error_helpers.cjs"); + +// Get the core object - in github-script context it's global, for testing we create a minimal version +const coreObj = + typeof core !== "undefined" + ? core + : { + info: msg => console.log(msg), + warning: msg => console.warn(msg), + error: msg => console.error(msg), + setFailed: msg => { + console.error(msg); + process.exitCode = 1; + }, + }; + +/** + * Parse the agent import specification to extract repository details + * Format: owner/repo/path@ref or owner/repo@ref or owner/repo/path + * @param {string} importSpec - The import specification + * @returns {{owner: string, repo: string, ref: string} | null} + */ +function parseAgentImportSpec(importSpec) { + if (!importSpec) { + return null; + } + + coreObj.info(`Parsing import spec: ${importSpec}`); + + // Remove section reference if present (file.md#Section) + let cleanSpec = importSpec; + if (importSpec.includes("#")) { + cleanSpec = importSpec.split("#")[0]; + } + + // Split on @ to get path and ref + const parts = cleanSpec.split("@"); + const pathPart = parts[0]; + const ref = parts.length > 1 ? parts[1] : "main"; + + // Parse path: owner/repo or owner/repo/path/to/file.md + const slashParts = pathPart.split("/"); + if (slashParts.length < 2) { + coreObj.warning(`Invalid import spec format: ${importSpec}`); + return null; + } + + const owner = slashParts[0]; + const repo = slashParts[1]; + + // Check if this is a local import (starts with . or doesn't have owner/repo format) + if (owner.startsWith(".") || owner.includes("github/workflows")) { + coreObj.info("Import is local, skipping remote .github folder merge"); + return null; + } + + coreObj.info(`Parsed: owner=${owner}, repo=${repo}, ref=${ref}`); + return { owner, repo, ref }; +} + +/** + * Check if a path exists + * @param {string} filePath - Path to check + * @returns {boolean} + */ +function pathExists(filePath) { + try { + fs.accessSync(filePath, fs.constants.F_OK); + return true; + } catch { + return false; + } +} + +/** + * Recursively get all files in a directory + * @param {string} dir - Directory to scan + * @param {string} baseDir - Base directory for relative paths + * @returns {string[]} Array of relative file paths + */ +function getAllFiles(dir, baseDir = dir) { + const files = []; + const items = fs.readdirSync(dir); + + for (const item of items) { + const fullPath = path.join(dir, item); + const stat = fs.statSync(fullPath); + + if (stat.isDirectory()) { + files.push(...getAllFiles(fullPath, baseDir)); + } else { + files.push(path.relative(baseDir, fullPath)); + } + } + + return files; +} + +/** + * Sparse checkout the .github folder from a remote repository + * @param {string} owner - Repository owner + * @param {string} repo - Repository name + * @param {string} ref - Git reference (branch, tag, or SHA) + * @param {string} tempDir - Temporary directory for checkout + */ +function sparseCheckoutGithubFolder(owner, repo, ref, tempDir) { + coreObj.info(`Performing sparse checkout of .github folder from ${owner}/${repo}@${ref}`); + + const repoUrl = `https://github.com/${owner}/${repo}.git`; + + try { + // Initialize git repository + execSync("git init", { cwd: tempDir, stdio: "pipe" }); + coreObj.info("Initialized temporary git repository"); + + // Configure sparse checkout + execSync("git config coreObj.sparseCheckout true", { cwd: tempDir, stdio: "pipe" }); + coreObj.info("Enabled sparse checkout"); + + // Set sparse checkout pattern to only include .github folder + const sparseCheckoutFile = path.join(tempDir, ".git", "info", "sparse-checkout"); + fs.writeFileSync(sparseCheckoutFile, ".github/\n"); + coreObj.info("Configured sparse checkout pattern: .github/"); + + // Add remote + execSync(`git remote add origin ${repoUrl}`, { cwd: tempDir, stdio: "pipe" }); + coreObj.info(`Added remote: ${repoUrl}`); + + // Fetch and checkout + coreObj.info(`Fetching ref: ${ref}`); + execSync(`git fetch --depth 1 origin ${ref}`, { cwd: tempDir, stdio: "pipe" }); + + coreObj.info("Checking out .github folder"); + execSync(`git checkout FETCH_HEAD`, { cwd: tempDir, stdio: "pipe" }); + + coreObj.info("Sparse checkout completed successfully"); + } catch (error) { + throw new Error(`Sparse checkout failed: ${getErrorMessage(error)}`); + } +} + +/** + * Merge .github folder from source to destination, failing on conflicts + * @param {string} sourcePath - Source .github folder path + * @param {string} destPath - Destination .github folder path + * @returns {{merged: number, conflicts: string[]}} + */ +function mergeGithubFolder(sourcePath, destPath) { + coreObj.info(`Merging .github folder from ${sourcePath} to ${destPath}`); + + const conflicts = []; + let mergedCount = 0; + + // Get all files from source .github folder + const sourceFiles = getAllFiles(sourcePath); + coreObj.info(`Found ${sourceFiles.length} files in source .github folder`); + + for (const relativePath of sourceFiles) { + const sourceFile = path.join(sourcePath, relativePath); + const destFile = path.join(destPath, relativePath); + + // Check if destination file exists + if (pathExists(destFile)) { + // Compare file contents + const sourceContent = fs.readFileSync(sourceFile); + const destContent = fs.readFileSync(destFile); + + if (!sourceContent.equals(destContent)) { + conflicts.push(relativePath); + coreObj.error(`Conflict detected: ${relativePath}`); + } else { + coreObj.info(`File already exists with same content: ${relativePath}`); + } + } else { + // Copy file to destination + const destDir = path.dirname(destFile); + if (!pathExists(destDir)) { + fs.mkdirSync(destDir, { recursive: true }); + coreObj.info(`Created directory: ${path.relative(destPath, destDir)}`); + } + + fs.copyFileSync(sourceFile, destFile); + mergedCount++; + coreObj.info(`Merged file: ${relativePath}`); + } + } + + return { merged: mergedCount, conflicts }; +} + +/** + * Merge a repository's .github folder into the workspace + * @param {string} owner - Repository owner + * @param {string} repo - Repository name + * @param {string} ref - Git reference + * @param {string} workspace - Workspace path + */ +async function mergeRepositoryGithubFolder(owner, repo, ref, workspace) { + coreObj.info(`Merging .github folder from ${owner}/${repo}@${ref} into workspace`); + + // Create temporary directory for sparse checkout + const tempDir = path.join("/tmp", `gh-aw-repo-merge-${Date.now()}`); + fs.mkdirSync(tempDir, { recursive: true }); + coreObj.info(`Created temporary directory: ${tempDir}`); + + try { + // Sparse checkout .github folder from remote repository + sparseCheckoutGithubFolder(owner, repo, ref, tempDir); + + // Check if .github folder exists in remote repository + const sourceGithubFolder = path.join(tempDir, ".github"); + if (!pathExists(sourceGithubFolder)) { + coreObj.warning(`Remote repository ${owner}/${repo}@${ref} does not contain a .github folder`); + return; + } + + // Merge .github folder into current repository + const destGithubFolder = path.join(workspace, ".github"); + + // Ensure destination .github folder exists + if (!pathExists(destGithubFolder)) { + fs.mkdirSync(destGithubFolder, { recursive: true }); + coreObj.info("Created .github folder in workspace"); + } + + const { merged, conflicts } = mergeGithubFolder(sourceGithubFolder, destGithubFolder); + + // Report results + if (conflicts.length > 0) { + coreObj.error(`Found ${conflicts.length} file conflicts:`); + for (const conflict of conflicts) { + coreObj.error(` - ${conflict}`); + } + throw new Error(`Cannot merge .github folder from ${owner}/${repo}@${ref}: ${conflicts.length} file(s) conflict with existing files`); + } + + if (merged > 0) { + coreObj.info(`Successfully merged ${merged} file(s) from ${owner}/${repo}@${ref}`); + } else { + coreObj.info("No new files to merge"); + } + } finally { + // Clean up temporary directory + if (pathExists(tempDir)) { + fs.rmSync(tempDir, { recursive: true, force: true }); + coreObj.info("Cleaned up temporary directory"); + } + } +} + +/** + * Main execution + */ +async function main() { + try { + coreObj.info("Starting remote .github folder merge"); + + // Check for repository imports (owner/repo@ref format - merge entire .github folder) + const repositoryImportsEnv = process.env.GH_AW_REPOSITORY_IMPORTS; + if (repositoryImportsEnv) { + coreObj.info(`Repository imports detected: ${repositoryImportsEnv}`); + + // Parse the JSON array of repository imports + let repositoryImports; + try { + repositoryImports = JSON.parse(repositoryImportsEnv); + } catch (error) { + throw new Error(`Failed to parse GH_AW_REPOSITORY_IMPORTS: ${getErrorMessage(error)}`); + } + + if (!Array.isArray(repositoryImports)) { + throw new Error("GH_AW_REPOSITORY_IMPORTS must be a JSON array"); + } + + // Get workspace path + const workspace = process.env.GITHUB_WORKSPACE; + if (!workspace) { + throw new Error("GITHUB_WORKSPACE environment variable not set"); + } + + // Process each repository import + for (const repoImport of repositoryImports) { + coreObj.info(`Processing repository import: ${repoImport}`); + + const parsed = parseAgentImportSpec(repoImport); + if (!parsed) { + coreObj.warning(`Skipping invalid repository import: ${repoImport}`); + continue; + } + + const { owner, repo, ref } = parsed; + await mergeRepositoryGithubFolder(owner, repo, ref, workspace); + } + + coreObj.info("All repository imports processed successfully"); + return; + } + + // Legacy path: Handle agent file imports (for backward compatibility) + // Get agent file path from environment + const agentFile = process.env.GH_AW_AGENT_FILE; + if (!agentFile) { + coreObj.info("No GH_AW_AGENT_FILE or GH_AW_REPOSITORY_IMPORTS specified, skipping .github folder merge"); + return; + } + + coreObj.info(`Agent file: ${agentFile}`); + + // Get agent import specification + const importSpec = process.env.GH_AW_AGENT_IMPORT_SPEC; + if (!importSpec) { + coreObj.info("No GH_AW_AGENT_IMPORT_SPEC specified, assuming local agent"); + return; + } + + coreObj.info(`Agent import spec: ${importSpec}`); + + // Parse import specification + const parsed = parseAgentImportSpec(importSpec); + if (!parsed) { + coreObj.info("Agent is local or import spec is invalid, skipping remote merge"); + return; + } + + const { owner, repo, ref } = parsed; + coreObj.info(`Remote agent detected: ${owner}/${repo}@${ref}`); + + // Get workspace path + const workspace = process.env.GITHUB_WORKSPACE; + if (!workspace) { + throw new Error("GITHUB_WORKSPACE environment variable not set"); + } + + await mergeRepositoryGithubFolder(owner, repo, ref, workspace); + + coreObj.info("Remote .github folder merge completed successfully"); + } catch (error) { + const errorMessage = getErrorMessage(error); + coreObj.setFailed(`Failed to merge remote .github folder: ${errorMessage}`); + } +} + +// Run if executed directly (not imported) +if (require.main === module) { + main(); +} + +module.exports = { + parseAgentImportSpec, + pathExists, + getAllFiles, + sparseCheckoutGithubFolder, + mergeGithubFolder, + mergeRepositoryGithubFolder, + main, +}; diff --git a/docs/src/content/docs/examples/agent-imports.md b/docs/src/content/docs/examples/agent-imports.md new file mode 100644 index 0000000000..e8221a4fac --- /dev/null +++ b/docs/src/content/docs/examples/agent-imports.md @@ -0,0 +1,206 @@ +--- +title: Agent Imports +description: Learn how to import and reuse specialized AI agents from external repositories to enhance your workflows with expert-crafted instructions and behavior. +sidebar: + order: 7 +--- + +GitHub Agentic Workflows supports importing agent files from external repositories, enabling you to reuse expert-crafted AI instructions and specialized behavior across teams and projects. + +## Why Import Agents? + +Agent files provide specialized AI instructions and behavior that can be shared and versioned like code. Importing agents from external repositories enables: + +- **Reuse expertise**: Leverage specialized agents created by experts +- **Consistency**: Ensure all teams use the same agent behavior +- **Versioning**: Pin to stable agent versions with semantic tags +- **Updates**: Update agent once, affects all workflows using it +- **Modularity**: Mix and match agents with workflow components + +## Basic Example + +### Importing a Code Review Agent + +Import a specialized code review agent from an external repository: + +```yaml wrap title=".github/workflows/pr-review.md" +--- +on: pull_request +engine: copilot +imports: + - acme-org/shared-agents/.github/agents/code-reviewer.md@v1.0.0 +permissions: + contents: read + pull-requests: write +--- + +# Automated Code Review + +Review this pull request for: +- Code quality and best practices +- Security vulnerabilities +- Performance issues +``` + +The agent file in `acme-org/shared-agents/.github/agents/code-reviewer.md` contains specialized instructions: + +```markdown title="acme-org/shared-agents/.github/agents/code-reviewer.md" +--- +name: Expert Code Reviewer +description: Specialized agent for comprehensive code review +tools: + github: + toolsets: [pull_requests, repos] +--- + +# Code Review Instructions + +You are an expert code reviewer with deep knowledge of: +- Security best practices (OWASP Top 10) +- Performance optimization patterns +- Code maintainability and readability + +When reviewing code: +1. Identify security vulnerabilities first +2. Check for performance issues +3. Ensure code follows team conventions +4. Suggest specific improvements with examples +``` + +## Versioning Agents + +Use semantic versioning to control agent updates: + +```yaml wrap +imports: + # Production - pin to specific version + - acme-org/ai-agents/.github/agents/security-auditor.md@v2.0.0 + + # Development - use latest + - acme-org/ai-agents/.github/agents/performance.md@main + + # Immutable - pin to commit SHA + - acme-org/ai-agents/.github/agents/custom.md@abc123def +``` + +## Agent Collections + +Organizations can create libraries of specialized agents: + +```text +acme-org/ai-agents/ +└── .github/ + └── agents/ + ├── code-reviewer.md # General code review + ├── security-auditor.md # Security-focused analysis + ├── performance-analyst.md # Performance optimization + ├── accessibility-checker.md # WCAG compliance + └── documentation-writer.md # Technical documentation +``` + +Teams import agents based on workflow needs: + +```yaml wrap title="Security-focused PR review" +--- +on: pull_request +engine: copilot +imports: + - acme-org/ai-agents/.github/agents/security-auditor.md@v2.0.0 + - acme-org/ai-agents/.github/agents/code-reviewer.md@v1.5.0 +--- + +# Security Review + +Perform comprehensive security review of this pull request. +``` + +```yaml wrap title="Accessibility-focused PR review" +--- +on: pull_request +engine: copilot +imports: + - acme-org/ai-agents/.github/agents/accessibility-checker.md@v1.0.0 +--- + +# Accessibility Review + +Check this pull request for WCAG 2.1 compliance issues. +``` + +## Combining Agents with Other Imports + +Mix agent imports with tool configurations and shared components: + +```yaml wrap +--- +on: pull_request +engine: copilot +imports: + # Import specialized agent + - acme-org/ai-agents/.github/agents/security-auditor.md@v2.0.0 + + # Import tool configurations + - acme-org/workflow-library/shared/tools/github-standard.md@v1.0.0 + + # Import MCP servers + - acme-org/workflow-library/shared/mcp/database.md@v1.0.0 + + # Import security policies + - acme-org/workflow-library/shared/config/security-policies.md@v1.0.0 +permissions: + contents: read + pull-requests: write +safe-outputs: + create-pull-request-review-comment: + max: 10 +--- + +# Comprehensive Security Review + +Perform detailed security analysis using specialized agent and tools. +``` + +## Caching and Offline Compilation + +Remote agent imports are cached in `.github/aw/imports/` by commit SHA: + +- **First compilation**: Downloads and caches the agent file +- **Subsequent compilations**: Uses cached file (works offline) +- **Cache sharing**: Same commit SHA across different refs shares cache +- **Version updates**: New versions download fresh agent files + +## Best Practices + +### For Agent Authors + +When creating shareable agents: + +1. **Use semantic versioning**: Tag releases with `v1.0.0`, `v1.1.0`, etc. +2. **Document capabilities**: Include clear frontmatter describing the agent +3. **Test thoroughly**: Validate agent behavior across different scenarios +4. **Maintain backwards compatibility**: Avoid breaking changes within major versions +5. **Provide examples**: Include usage examples in the repository + +### For Agent Users + +When importing agents: + +1. **Pin production to versions**: Use `@v1.0.0` for stable production workflows +2. **Test updates first**: Try new versions in non-production workflows +3. **Use branches for development**: Import from `@main` during active development +4. **Review agent code**: Understand what the agent does before importing +5. **Monitor updates**: Subscribe to releases for agent repositories + +## Constraints + +- **One agent per workflow**: Only one agent file can be imported per workflow +- **Agent path detection**: Files in `.github/agents/` are automatically recognized +- **Local or remote**: Can import from local `.github/agents/` or remote repositories +- **Same format**: Remote and local agents use identical file format + +## Related Documentation + +- [Custom Agents Reference](/gh-aw/reference/custom-agents/) - Agent file format and requirements +- [Imports Reference](/gh-aw/reference/imports/) - Complete import system documentation +- [Packaging & Distribution](/gh-aw/guides/packaging-imports/) - Managing workflow imports +- [Frontmatter](/gh-aw/reference/frontmatter/) - Configuration options reference diff --git a/docs/src/content/docs/guides/packaging-imports.md b/docs/src/content/docs/guides/packaging-imports.md index 4cec57e819..a8c31d6a8b 100644 --- a/docs/src/content/docs/guides/packaging-imports.md +++ b/docs/src/content/docs/guides/packaging-imports.md @@ -114,6 +114,85 @@ safe-outputs: See [Imports Reference](/gh-aw/reference/imports/) for complete merge semantics. +## Importing Agents from Repositories + +Agent files provide specialized AI instructions and behavior. You can import agents from external repositories to reuse expert-crafted prompts and configurations across teams and projects. + +### Creating a Shareable Agent + +Store agent files in `.github/agents/` of any repository: + +```markdown title="acme-org/ai-agents/.github/agents/code-reviewer.md" +--- +name: Expert Code Reviewer +description: Specialized agent for comprehensive code review with security focus +tools: + github: + toolsets: [pull_requests, repos] +--- + +# Code Review Instructions + +You are an expert code reviewer with deep knowledge of: +- Security best practices and OWASP guidelines +- Performance optimization patterns +- Code maintainability and readability +- Testing strategies and coverage + +When reviewing code: +1. Identify security vulnerabilities first +2. Check for performance issues +3. Ensure code follows team conventions +4. Suggest specific improvements with examples +``` + +### Importing Remote Agents + +Reference the agent using `owner/repo/path@version` format: + +```yaml wrap +--- +on: pull_request +engine: copilot +imports: + - acme-org/ai-agents/.github/agents/code-reviewer.md@v1.2.0 +permissions: + contents: read + pull-requests: write +--- + +# PR Review Workflow +The code reviewer agent will analyze this pull request and provide detailed feedback. +``` + +### Agent Versioning + +Use semantic versioning for production stability: + +```yaml wrap +imports: + - acme-org/ai-agents/.github/agents/security-auditor.md@v2.0.0 # Production + - acme-org/ai-agents/.github/agents/performance.md@main # Latest + - acme-org/ai-agents/.github/agents/custom.md@abc123def # Immutable +``` + +### Agent Collections + +Organizations can maintain libraries of specialized agents: + +``` +acme-org/ai-agents/ +└── .github/ + └── agents/ + ├── code-reviewer.md # General code review + ├── security-auditor.md # Security-focused analysis + ├── performance-analyst.md # Performance optimization + ├── accessibility-checker.md # WCAG compliance + └── documentation-writer.md # Technical documentation +``` + +Teams import agents based on workflow needs, with all agents benefiting from centralized updates and versioning. + ## Example: Modular Workflow with Imports Create a shared Model Context Protocol (MCP) server configuration in `.github/workflows/shared/mcp/tavily.md`: @@ -159,29 +238,35 @@ A development team can create a shared configuration repository with reusable co ``` acme-org/workflow-library/ +├── .github/ +│ └── agents/ +│ ├── code-reviewer.md # Specialized code review agent +│ ├── security-auditor.md # Security-focused agent +│ └── performance-analyst.md # Performance optimization agent ├── shared/ │ ├── tools/ -│ │ ├── github-standard.md # Standard GitHub API toolsets -│ │ └── code-analysis.md # Code quality tools +│ │ ├── github-standard.md # Standard GitHub API toolsets +│ │ └── code-analysis.md # Code quality tools │ ├── mcp/ -│ │ ├── tavily.md # Web search -│ │ └── database.md # Database access +│ │ ├── tavily.md # Web search +│ │ └── database.md # Database access │ └── config/ -│ ├── security-policies.md # Security constraints -│ └── notification-setup.md # Notification settings +│ ├── security-policies.md # Security constraints +│ └── notification-setup.md # Notification settings └── workflows/ ├── issue-triage.md ├── pr-review.md └── release-automation.md ``` -Individual workflows import required components: +Individual workflows import required components and agents: ```yaml wrap --- on: pull_request engine: copilot imports: + - acme-org/workflow-library/.github/agents/code-reviewer.md@v2.0.0 - acme-org/workflow-library/shared/tools/github-standard.md@v2.0.0 - acme-org/workflow-library/shared/tools/code-analysis.md@v2.0.0 - acme-org/workflow-library/shared/config/security-policies.md@v2.0.0 @@ -191,29 +276,38 @@ safe-outputs: --- # Code Review Agent -Automated code review with security policy enforcement. +Automated code review with security policy enforcement using shared agent. ``` **Benefits**: -- **Consistency**: All workflows use same tool configurations +- **Consistency**: All workflows use same tool configurations and agent behavior - **Maintainability**: Update imports once, affects all workflows - **Versioning**: Pin to stable versions with semantic tags -- **Modularity**: Mix and match components as needed -- **Governance**: Security policies enforced through imports +- **Modularity**: Mix and match components and agents as needed +- **Governance**: Security policies and review standards enforced through imports +- **Agent Sharing**: Reuse specialized agent instructions across teams and repositories ## Specification Formats and Validation -Workflow specifications require minimum 3 parts (owner/repo/workflow-name) for short form. Explicit paths must end with `.md`. Versions can be semantic tags (`@v1.0.0`), branches (`@develop`), or commit SHAs. Identifiers use alphanumeric characters with hyphens/underscores (cannot start/end with hyphen). +Workflow and import specifications require minimum 3 parts (owner/repo/path) for remote imports. Explicit paths must end with `.md`. Versions can be semantic tags (`@v1.0.0`), branches (`@develop`), or commit SHAs. Identifiers use alphanumeric characters with hyphens/underscores (cannot start/end with hyphen). **Examples:** - Repository: `owner/repo[@version]` - Short workflow: `owner/repo/workflow[@version]` (adds `workflows/` prefix and `.md`) - Explicit workflow: `owner/repo/path/to/workflow.md[@version]` +- Agent import: `owner/repo/.github/agents/agent-name.md[@version]` +- Shared import: `owner/repo/shared/tools/config.md[@version]` - GitHub URL: `https://github.com/owner/repo/blob/main/workflows/ci-doctor.md` - Raw URL: `https://raw.githubusercontent.com/owner/repo/refs/heads/main/workflows/ci-doctor.md` ## Best Practices -Use semantic versioning for stable workflows, branches for development, and commit SHAs for immutability. Organize reusable components in a `shared/` directory with descriptive names. Review updates with `--verbose` before applying, test on branches, and keep local modifications minimal to reduce merge conflicts. +Use semantic versioning for stable workflows and agents, branches for development, and commit SHAs for immutability. Organize reusable components in `shared/` directories and agent files in `.github/agents/` with descriptive names. Review updates with `--verbose` before applying, test on branches, and keep local modifications minimal to reduce merge conflicts. + +When sharing agents across teams: +- Use semantic versioning for production agents (`@v1.0.0`) +- Document agent capabilities and requirements in frontmatter +- Test agent updates in non-production workflows first +- Maintain backwards compatibility within major versions -**Related:** [CLI Commands](/gh-aw/setup/cli/) | [Workflow Structure](/gh-aw/reference/workflow-structure/) | [Frontmatter](/gh-aw/reference/frontmatter/) | [Imports](/gh-aw/reference/imports/) +**Related:** [CLI Commands](/gh-aw/setup/cli/) | [Workflow Structure](/gh-aw/reference/workflow-structure/) | [Frontmatter](/gh-aw/reference/frontmatter/) | [Imports](/gh-aw/reference/imports/) | [Custom Agents](/gh-aw/reference/custom-agents/) diff --git a/docs/src/content/docs/reference/custom-agents.md b/docs/src/content/docs/reference/custom-agents.md index 06fad3e042..3619cc3289 100644 --- a/docs/src/content/docs/reference/custom-agents.md +++ b/docs/src/content/docs/reference/custom-agents.md @@ -27,7 +27,11 @@ You are a specialized code review agent. Focus on: ## Using Custom Agents -Import the agent file in your workflow using the `imports` field: +Import agent files in your workflow using the `imports` field. Agents can be imported from local `.github/agents/` directories or from external repositories. + +### Local Agent Import + +Import an agent from your repository: ```yaml wrap --- @@ -40,14 +44,35 @@ imports: Review the pull request and provide feedback. ``` +### Remote Agent Import + +Import an agent from an external repository using the `owner/repo/path@ref` format: + +```yaml wrap +--- +on: pull_request +engine: copilot +imports: + - acme-org/shared-agents/.github/agents/code-reviewer.md@v1.0.0 +--- + +Perform comprehensive code review using shared agent instructions. +``` + +Remote agent imports support versioning: +- **Semantic tags**: `@v1.0.0` (recommended for production) +- **Branch names**: `@main`, `@develop` (for development) +- **Commit SHAs**: `@abc123def` (for immutable references) + The agent instructions are merged with the workflow prompt, customizing the AI engine's behavior for specific tasks. ## Agent File Requirements -- **Location**: Must be in `.github/agents/` directory +- **Location**: Must be in a `.github/agents/` directory (local or remote repository) - **Format**: Markdown with YAML frontmatter - **Frontmatter**: Can include `name`, `description`, `tools`, and `mcp-servers` - **One per workflow**: Only one agent file can be imported per workflow +- **Caching**: Remote agents are cached by commit SHA in `.github/aw/imports/` ## Built-in Agents diff --git a/docs/src/content/docs/reference/imports.md b/docs/src/content/docs/reference/imports.md index 46b02df51b..794a14d75f 100644 --- a/docs/src/content/docs/reference/imports.md +++ b/docs/src/content/docs/reference/imports.md @@ -75,7 +75,11 @@ Remote imports are cached in `.github/aw/imports/` to enable offline compilation ## Agent Files -Import custom agent files from `.github/agents/` to customize AI engine behavior. Agent files are markdown documents with specialized instructions that modify how the AI interprets and executes workflows. +Import custom agent files to customize AI engine behavior. Agent files are markdown documents with specialized instructions that modify how the AI interprets and executes workflows. Agent files can be imported from local `.github/agents/` directories or from external repositories. + +### Local Agent Imports + +Import agent files from your repository's `.github/agents/` directory: ```yaml wrap --- @@ -86,7 +90,33 @@ imports: --- ``` -Only one agent file can be imported per workflow. +### Remote Agent Imports + +Import agent files from external repositories using the `owner/repo/path@ref` format: + +```yaml wrap +--- +on: pull_request +engine: copilot +imports: + - githubnext/shared-agents/.github/agents/security-reviewer.md@v1.0.0 +--- + +# PR Security Review + +Analyze pull requests for security vulnerabilities using the shared security reviewer agent. +``` + +Remote agent imports support the same versioning as other imports: +- Semantic tags: `@v1.0.0`, `@v2.1.3` +- Branch names: `@main`, `@develop` +- Commit SHAs: `@abc123def456` (immutable references) + +### Constraints + +- **One agent per workflow**: Only one agent file can be imported per workflow (local or remote) +- **Agent path detection**: Files in `.github/agents/` directories are automatically recognized as agent files +- **Caching**: Remote agents are cached in `.github/aw/imports/` by commit SHA, enabling offline compilation ## Frontmatter Merging diff --git a/pkg/parser/import_processor.go b/pkg/parser/import_processor.go index 39e509e3b9..5a9c513d50 100644 --- a/pkg/parser/import_processor.go +++ b/pkg/parser/import_processor.go @@ -34,6 +34,8 @@ type ImportsResult struct { MergedJobs string // Merged jobs from imported YAML workflows (JSON format) ImportedFiles []string // List of imported file paths (for manifest) AgentFile string // Path to custom agent file (if imported) + AgentImportSpec string // Original import specification for agent file (e.g., "owner/repo/path@ref") + RepositoryImports []string // List of repository imports (format: "owner/repo@ref") for .github folder merging // ImportInputs uses map[string]any because input values can be different types (string, number, boolean). // This is parsed from YAML frontmatter where the structure is dynamic and not known at compile time. // This is an appropriate use of 'any' for dynamic YAML/JSON data. @@ -181,11 +183,22 @@ func processImportsFromFrontmatterWithManifestAndSource(frontmatter map[string]a var caches []string // Track cache configurations (appended in order) var jobsBuilder strings.Builder // Track jobs from imported YAML workflows var agentFile string // Track custom agent file + var agentImportSpec string // Track agent import specification for remote imports + var repositoryImports []string // Track repository-only imports for .github folder merging importInputs := make(map[string]any) // Aggregated input values from all imports // Seed the queue with initial imports for _, importSpec := range importSpecs { importPath := importSpec.Path + + // Check if this is a repository-only import (owner/repo@ref without file path) + if isRepositoryImport(importPath) { + log.Printf("Detected repository import: %s", importPath) + repositoryImports = append(repositoryImports, importPath) + // Repository imports don't need further processing - they're handled at runtime + continue + } + // Handle section references (file.md#Section) var filePath, sectionName string if strings.Contains(importPath, "#") { @@ -280,6 +293,11 @@ func processImportsFromFrontmatterWithManifestAndSource(frontmatter map[string]a } log.Printf("Found agent file: %s (resolved to: %s)", item.fullPath, agentFile) + // Store the original import specification for remote agents + // This allows runtime detection and .github folder merging + agentImportSpec = item.importPath + log.Printf("Agent import specification: %s", agentImportSpec) + // For agent files, only extract markdown content markdownContent, err := processIncludedFileWithVisited(item.fullPath, item.sectionName, false, visited) if err != nil { @@ -567,6 +585,8 @@ func processImportsFromFrontmatterWithManifestAndSource(frontmatter map[string]a MergedJobs: jobsBuilder.String(), ImportedFiles: topologicalOrder, AgentFile: agentFile, + AgentImportSpec: agentImportSpec, + RepositoryImports: repositoryImports, ImportInputs: importInputs, }, nil } diff --git a/pkg/parser/remote_fetch.go b/pkg/parser/remote_fetch.go index 563cdb5dde..b076fe9e39 100644 --- a/pkg/parser/remote_fetch.go +++ b/pkg/parser/remote_fetch.go @@ -50,6 +50,58 @@ func isCustomAgentFile(filePath string) bool { return strings.Contains(normalizedPath, ".github/agents/") && strings.HasSuffix(strings.ToLower(normalizedPath), ".md") } +// isRepositoryImport checks if an import spec is a repository-only import (no file path) +// Format: owner/repo@ref or owner/repo (downloads entire .github folder, no agent extraction) +func isRepositoryImport(importPath string) bool { + // Remove section reference if present + cleanPath := importPath + if idx := strings.Index(importPath, "#"); idx != -1 { + cleanPath = importPath[:idx] + } + + // Remove ref if present to check the path structure + pathWithoutRef := cleanPath + if idx := strings.Index(cleanPath, "@"); idx != -1 { + pathWithoutRef = cleanPath[:idx] + } + + // Split by slash to count parts + parts := strings.Split(pathWithoutRef, "/") + + // Repository import has exactly 2 parts: owner/repo + // File imports have 1 part (local file) or 3+ parts (owner/repo/path/to/file) + if len(parts) != 2 { + return false + } + + // Reject local paths + if strings.HasPrefix(pathWithoutRef, ".") || strings.HasPrefix(pathWithoutRef, "/") { + return false + } + + // Reject paths that start with common local directory names + if strings.HasPrefix(pathWithoutRef, "shared/") { + return false + } + + // Additional validation: check if it looks like a valid owner/repo format + // GitHub identifiers can't start with numbers, must be alphanumeric with hyphens/underscores + owner := parts[0] + repo := parts[1] + + // Basic validation - ensure they're not empty and don't look like file extensions + if owner == "" || repo == "" { + return false + } + + // Reject if repo part looks like a file extension (ends with .md, .yaml, etc.) + if strings.Contains(repo, ".") { + return false + } + + return true +} + // ResolveIncludePath resolves include path based on workflowspec format or relative path func ResolveIncludePath(filePath, baseDir string, cache *ImportCache) (string, error) { remoteLog.Printf("Resolving include path: file_path=%s, base_dir=%s", filePath, baseDir) diff --git a/pkg/workflow/compiler_orchestrator_workflow.go b/pkg/workflow/compiler_orchestrator_workflow.go index 323ad641c1..5155e6d2d5 100644 --- a/pkg/workflow/compiler_orchestrator_workflow.go +++ b/pkg/workflow/compiler_orchestrator_workflow.go @@ -119,6 +119,8 @@ func (c *Compiler) buildInitialWorkflowData( AI: engineSetup.engineSetting, EngineConfig: engineSetup.engineConfig, AgentFile: importsResult.AgentFile, + AgentImportSpec: importsResult.AgentImportSpec, + RepositoryImports: importsResult.RepositoryImports, NetworkPermissions: engineSetup.networkPermissions, SandboxConfig: applySandboxDefaults(engineSetup.sandboxConfig, engineSetup.engineConfig), NeedsTextOutput: toolsResult.needsTextOutput, diff --git a/pkg/workflow/compiler_types.go b/pkg/workflow/compiler_types.go index 84820da5de..755c30bddb 100644 --- a/pkg/workflow/compiler_types.go +++ b/pkg/workflow/compiler_types.go @@ -394,6 +394,8 @@ type WorkflowData struct { AI string // "claude" or "codex" (for backwards compatibility) EngineConfig *EngineConfig // Extended engine configuration AgentFile string // Path to custom agent file (from imports) + AgentImportSpec string // Original import specification for agent file (e.g., "owner/repo/path@ref") + RepositoryImports []string // Repository-only imports (format: "owner/repo@ref") for .github folder merging StopTime string SkipIfMatch *SkipIfMatchConfig // skip-if-match configuration with query and max threshold SkipIfNoMatch *SkipIfNoMatchConfig // skip-if-no-match configuration with query and min threshold diff --git a/pkg/workflow/compiler_yaml_main_job.go b/pkg/workflow/compiler_yaml_main_job.go index 239a542eef..6bd9071a35 100644 --- a/pkg/workflow/compiler_yaml_main_job.go +++ b/pkg/workflow/compiler_yaml_main_job.go @@ -1,6 +1,7 @@ package workflow import ( + "encoding/json" "fmt" "strings" ) @@ -35,6 +36,40 @@ func (c *Compiler) generateMainJobSteps(yaml *strings.Builder, data *WorkflowDat } } + // Add merge remote .github folder step for repository imports or agent imports + needsGithubMerge := (len(data.RepositoryImports) > 0) || (data.AgentFile != "" && data.AgentImportSpec != "") + if needsGithubMerge { + compilerYamlLog.Printf("Adding merge remote .github folder step") + yaml.WriteString(" - name: Merge remote .github folder\n") + yaml.WriteString(" if: ${{ always() }}\n") + fmt.Fprintf(yaml, " uses: %s\n", GetActionPin("actions/github-script")) + yaml.WriteString(" env:\n") + + // Set repository imports if present + if len(data.RepositoryImports) > 0 { + // Convert to JSON array for the script + repoImportsJSON, err := json.Marshal(data.RepositoryImports) + if err != nil { + compilerYamlLog.Printf("Warning: failed to marshal repository imports: %v", err) + } else { + fmt.Fprintf(yaml, " GH_AW_REPOSITORY_IMPORTS: '%s'\n", string(repoImportsJSON)) + } + } + + // Set agent import spec if present (legacy path) + if data.AgentFile != "" && data.AgentImportSpec != "" { + fmt.Fprintf(yaml, " GH_AW_AGENT_FILE: \"%s\"\n", data.AgentFile) + fmt.Fprintf(yaml, " GH_AW_AGENT_IMPORT_SPEC: \"%s\"\n", data.AgentImportSpec) + } + + yaml.WriteString(" with:\n") + yaml.WriteString(" script: |\n") + yaml.WriteString(" const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs');\n") + yaml.WriteString(" setupGlobals(core, github, context, exec, io);\n") + yaml.WriteString(" const { main } = require('/opt/gh-aw/actions/merge_remote_agent_github_folder.cjs');\n") + yaml.WriteString(" await main();\n") + } + // Add automatic runtime setup steps if needed // This detects runtimes from custom steps and MCP configs runtimeRequirements := DetectRuntimeRequirements(data)