diff --git a/.github/workflows/agent-ci.yml b/.github/workflows/agent-ci.yml new file mode 100644 index 0000000000..ee11faaec6 --- /dev/null +++ b/.github/workflows/agent-ci.yml @@ -0,0 +1,1418 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + paths: + - '**.go' + - 'pkg/workflow/**' + - 'actions/**' + - '.github/workflows/ci.yml' + - '.github/workflows/**/*.md' + workflow_dispatch: +jobs: + test: + runs-on: ubuntu-latest + permissions: + contents: read + concurrency: + group: ci-${{ github.ref }}-test + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6 + with: + go-version-file: go.mod + cache: true + + - name: Report Go cache status + run: | + if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then + echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY + fi + + - name: Verify dependencies + run: go mod verify + + - name: Run unit tests with coverage + run: | + set -o pipefail + go test -v -parallel=8 -timeout=3m -run='^Test' -coverprofile=coverage.out -json ./... | tee test-result-unit.json + go tool cover -html=coverage.out -o coverage.html + + # Coverage reports for recent builds only - 7 days is sufficient for debugging recent changes + - name: Upload coverage report + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: coverage-report + path: coverage.html + retention-days: 7 + + - name: Upload unit test results + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: test-result-unit + path: test-result-unit.json + retention-days: 14 + + integration: + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + test-group: + - name: "CLI Compile & Poutine" + packages: "./pkg/cli" + pattern: "^TestCompile[^W]|TestPoutine|TestBuildCommandString|TestBuildSourceString|TestBuildWorkflowDescription|TestPrintCompilationSummary|TestCompilationStats|TestModifyWorkflowForTrialMode" # Exclude TestCompileWorkflows to avoid duplicates + - name: "CLI MCP Playwright" + packages: "./pkg/cli" + pattern: "TestMCPInspectPlaywright" + - name: "CLI MCP Gateway" + packages: "./pkg/cli" + pattern: "TestMCPGateway" + - name: "CLI MCP Other" + packages: "./pkg/cli" + pattern: "TestMCPAdd|TestMCPInspectGitHub|TestMCPServer|TestMCPConfig|TestMCPCommand|TestIsRunningAsMCPServer|TestHeaderRoundTripper" + - name: "CLI Logs & Firewall" + packages: "./pkg/cli" + pattern: "TestLogs|TestFirewall|TestNoStopTime|TestLocalWorkflow" + - name: "CLI Progress Flag" # Isolate slow test (~65s for TestProgressFlagSignature) + packages: "./pkg/cli" + pattern: "TestProgressFlagSignature" + - name: "CLI HTTP MCP Connect" # Isolate slow HTTP MCP connection tests (~43s) + packages: "./pkg/cli" + pattern: "TestConnectHTTPMCPServer" + - name: "CLI Compile Workflows" # Isolate slow workflow compilation test + packages: "./pkg/cli" + pattern: "TestCompileWorkflows|TestCollectWorkflowFiles|TestFilterWorkflowFiles|TestNormalizeWorkflowFile" + - name: "CLI Security Tools" # Group security tool compilation tests + packages: "./pkg/cli" + pattern: "TestCompileWithZizmor|TestCompileWithPoutine|TestCompileWithPoutineAndZizmor|TestParseAndDisplayZizmorOutput|TestSecurityToolsIndependentOfValidate|TestSecurity" + - name: "CLI Add & List Commands" + packages: "./pkg/cli" + pattern: "^TestAdd|^TestList" + - name: "CLI Update Command" + packages: "./pkg/cli" + pattern: "^TestUpdate" + - name: "CLI Audit & Inspect" + packages: "./pkg/cli" + pattern: "^TestAudit|^TestInspect" + - name: "CLI Docker Build" # Isolate slow Docker build test (~38s) + packages: "./pkg/cli" + pattern: "TestDockerBuild|TestDockerfile|TestDockerImage" + - name: "CLI Shell Completion" # Shell completion tests + packages: "./pkg/cli" + pattern: "TestCompletion|TestUninstall.*Completion" + - name: "CLI Secrets & Encryption" # Secrets and encryption tests + packages: "./pkg/cli" + pattern: "TestSecret|TestEncrypt|TestCheckSecretsAvailability|TestResolveSecretValueForSet|TestExtractSecretName|TestExtractSecretsFromConfig|TestNewSecretsCommand" + - name: "CLI Status & Versioning" # Status and version command tests + packages: "./pkg/cli" + pattern: "TestStatus|TestVersion|TestWorkflowStatus|TestWorkflowRunInfo|TestShellTypeString" + - name: "CLI Update & Upgrade" # Update and upgrade command tests + packages: "./pkg/cli" + pattern: "TestUpdate|TestUpgrade|TestCheckForUpdates|TestShowUpdateSummary|TestMajorVersionPreference|TestPreciseVersionPreference|TestActionKeyVersionConsistency|TestMarshalActionsLockSorted|TestShouldCheckForUpdate|TestGetLastCheckFilePath|TestEnsureUpgradeAgenticWorkflowAgent|TestEnsureUpgradeAgenticWorkflowsPrompt" + - name: "CLI Workflows Path & Discovery" # Workflow path resolution and discovery tests + packages: "./pkg/cli" + pattern: "TestFindRunnableWorkflows|TestFindWorkflowsWithSource|TestGetAvailableWorkflowNames|TestGetWorkflowInputs|TestGetLockFilePath|TestGetMarkdownWorkflowFilesExcludesREADME|TestReadWorkflowFile|TestResolveWorkflowPath|TestIsWorkflowFile|TestWorkflowCounting|TestWorkflowResolutionWindowsCompatibility|TestSuggestWorkflowNames|TestValidateWorkflowName|TestValidateWorkflowIntent|TestValidateWorkflowInputs|TestWorkflowSpecString" + - name: "CLI Trial & Interactive" # Trial mode and interactive tests + packages: "./pkg/cli" + pattern: "TestTrial|TestLipglossImportPresent|TestSectionCompositionPattern" + - name: "CLI Run Command" # Run command tests + packages: "./pkg/cli" + pattern: "TestRun|TestPoll|TestTrackWorkflowFailure|TestInputValidation|TestInputFlagSignature|TestRefFlagSignature" + - name: "CLI Repo & Git" # Repository and git related tests + packages: "./pkg/cli" + pattern: "TestRepo|TestGet.*RepoSlug|TestExtractBaseRepo|TestExtractIssueNumberFromURL|TestHostRepoSlugProcessing|TestHasLocalModifications|TestPushWorkflowFiles|TestCloneRepoWithVersion|TestClearCurrentRepoSlugCache|TestIsAuthenticationError|TestRemoveWorkflows_KeepOrphansFlag" + - name: "CLI Spec Parsing" # Spec parsing tests + packages: "./pkg/cli" + pattern: "TestParse.*Spec|TestResolveImportPathLocal|TestIsWorkflowSpecFormatLocal|TestResolveLatestRef|TestParseAwInfo|TestParseVersion" + - name: "CLI Validators & Semver" # Validator and semver tests + packages: "./pkg/cli" + pattern: "TestValidat|TestIsCommitSHA|TestIsPreciseVersion|TestIsSemanticVersionTag|TestIsRunnable" + - name: "CLI Tool Graph" # Tool graph tests + packages: "./pkg/cli" + pattern: "TestToolGraph" + - name: "CLI Signal Handling" # Signal handling tests + packages: "./pkg/cli" + pattern: "TestPollWithSignalHandling" + - name: "CLI Makefile & Setup" # Makefile and setup action tests + packages: "./pkg/cli" + pattern: "TestMakefile|TestSetupCLIAction" + - name: "CLI Merge Content" # Workflow content merging tests + packages: "./pkg/cli" + pattern: "TestMergeWorkflowContent" + - name: "CLI Workflows Actions" # Workflow actions and action extraction tests + packages: "./pkg/cli" + pattern: "TestGetActionSHAForTag|TestExtractNpxPackages" + - name: "CLI Codespace Integration" # Codespace specific tests + packages: "./pkg/cli" + pattern: "TestCodespace|TestDetectShell" + - name: "CLI Safe Inputs" # Safe inputs tests + packages: "./pkg/cli" + pattern: "TestSafeInputsMCPServerCompilation" + - name: "CMD Main & Entry" # CMD tests for main entry point + packages: "./cmd/gh-aw" + pattern: "TestMain|TestCommandGroup|TestCommandLine|TestCommand|TestArgument|TestInitFunction|TestVersion|TestMCPCommand|TestMCPSubcommand|TestCampaignSubcommand|TestPRSubcommand|TestNoCommandsInAdditionalCommandsWithGroups" + - name: "CLI Completion & Other" # Remaining catch-all (reduced from original) + packages: "./pkg/cli" + pattern: "" # Catch-all for tests not matched by other CLI patterns + skip_pattern: "^TestCompile|TestPoutine|TestMCPInspectPlaywright|TestMCPGateway|TestMCPAdd|TestMCPInspectGitHub|TestMCPServer|TestMCPConfig|TestMCPCommand|TestIsRunningAsMCPServer|TestHeaderRoundTripper|TestLogs|TestFirewall|TestNoStopTime|TestLocalWorkflow|TestProgressFlagSignature|TestConnectHTTPMCPServer|TestCompileWorkflows|TestCollectWorkflowFiles|TestFilterWorkflowFiles|TestNormalizeWorkflowFile|TestParseAndDisplayZizmorOutput|TestSecurityToolsIndependentOfValidate|TestSecurity|^TestAdd|^TestList|^TestUpdate|^TestUpgrade|TestCheckForUpdates|TestShowUpdateSummary|TestMajorVersionPreference|TestPreciseVersionPreference|TestActionKeyVersionConsistency|TestMarshalActionsLockSorted|^TestAudit|^TestInspect|TestDockerBuild|TestDockerfile|TestDockerImage|TestCompletion|TestUninstall.*Completion|TestSecret|TestEncrypt|TestCheckSecretsAvailability|TestResolveSecretValueForSet|TestStatus|TestVersion|TestWorkflowStatus|TestWorkflowRunInfo|TestShellTypeString|TestFindRunnableWorkflows|TestFindWorkflowsWithSource|TestGetAvailableWorkflowNames|TestGetWorkflowInputs|TestGetLockFilePath|TestGetMarkdownWorkflowFilesExcludesREADME|TestReadWorkflowFile|TestResolveWorkflowPath|TestIsWorkflowFile|TestWorkflowCounting|TestWorkflowResolutionWindowsCompatibility|TestSuggestWorkflowNames|TestValidateWorkflowName|TestValidateWorkflowIntent|TestValidateWorkflowInputs|TestWorkflowSpecString|TestTrial|TestLipglossImportPresent|TestSectionCompositionPattern|TestRun|TestPoll|TestTrackWorkflowFailure|TestInputValidation|TestInputFlagSignature|TestRefFlagSignature|TestRepo|TestGet.*RepoSlug|TestExtractBaseRepo|TestExtractIssueNumberFromURL|TestHostRepoSlugProcessing|TestHasLocalModifications|TestPushWorkflowFiles|TestCloneRepoWithVersion|TestParse.*Spec|TestResolveImportPathLocal|TestIsWorkflowSpecFormatLocal|TestResolveLatestRef|TestValidat|TestIsCommitSHA|TestIsPreciseVersion|TestIsSemanticVersionTag|TestIsRunnable|TestToolGraph|TestPollWithSignalHandling|TestMakefile|TestSetupCLIAction|TestMergeWorkflowContent|TestGetActionSHAForTag|TestExtractNpxPackages|TestCodespace|TestDetectShell|TestSafeInputsMCPServerCompilation|TestBuildCommandString|TestBuildSourceString|TestBuildWorkflowDescription|TestPrintCompilationSummary|TestCompilationStats|TestModifyWorkflowForTrialMode" + - name: "Workflow Compiler" + packages: "./pkg/workflow" + pattern: "TestCompile|TestWorkflow|TestGenerate|TestParse" + - name: "Workflow Tools & MCP" + packages: "./pkg/workflow" + pattern: "TestMCP|TestTool|TestSkill|TestPlaywright|TestFirewall" + - name: "Workflow Validation" + packages: "./pkg/workflow" + pattern: "TestValidat|TestLock|TestError|TestWarning|TestRepositoryFeaturesValidation|TestBackwardCompatibilityWithClaudeFormat" + - name: "Workflow Safe Outputs" + packages: "./pkg/workflow" + pattern: "SafeOutputs|SafeInputs|CreatePullRequest|OutputLabel|HasSafeOutputs|ConsolidatedSafeOutputs|SafeOutputJobs" + - name: "Workflow GitHub & Git" + packages: "./pkg/workflow" + pattern: "GitHub|Git|PushToPullRequest|BuildFromAllowed|AdditionalClaudeTools|CopilotGitCommands" + - name: "Workflow Rendering & Bundling" + packages: "./pkg/workflow" + pattern: "Render|Bundle|Script|WritePromptText" + - name: "Workflow Cache" + packages: "./pkg/workflow" + pattern: "^TestCache|TestCacheDependencies|TestCacheKey|TestValidateCache" + - name: "Workflow Actions Pin Validation" + packages: "./pkg/workflow" + pattern: "^TestActionPinSHAsMatchVersionTags" + - name: "Workflow Actions & Containers" + packages: "./pkg/workflow" + pattern: "^TestAction[^P]|Container" + - name: "Workflow Dependabot & Security" + packages: "./pkg/workflow" + pattern: "Dependabot|Security|PII" + - name: "CMD Tests" # All cmd/gh-aw integration tests + packages: "./cmd/gh-aw" + pattern: "" + skip_pattern: "TestMain|TestCommandGroup|TestCommandLine|TestCommand|TestArgument|TestInitFunction|TestVersion|TestMCPCommand|TestMCPSubcommand|TestCampaignSubcommand|TestPRSubcommand|TestNoCommandsInAdditionalCommandsWithGroups" # These are in the "CMD Main & Entry" group above + - name: "Parser Remote Fetch & Cache" + packages: "./pkg/parser" + pattern: "TestDownloadFileFromGitHub|TestResolveIncludePath|TestDownloadIncludeFromWorkflowSpec|TestImportCache|TestFrontmatterLocation|TestFrontmatterOffsetCalculation|TestImprovementComparison" + - name: "Parser Location & Validation" + packages: "./pkg/parser" + pattern: "" # Catch-all for tests not matched by other Parser patterns + skip_pattern: "TestDownloadFileFromGitHub|TestResolveIncludePath|TestDownloadIncludeFromWorkflowSpec|TestImportCache|TestFrontmatterLocation|TestFrontmatterOffsetCalculation|TestImprovementComparison" + - name: "Workflow Permissions" + packages: "./pkg/workflow" + pattern: "TestPermissions|TestPackageExtractor|TestCollectPackagesFromWorkflow|TestExtractNpx" + - name: "Workflow Misc Part 1" # Split large catch-all into two balanced groups + packages: "./pkg/workflow" + pattern: "TestAgent|TestCopilot|TestCustomEngine|TestEngine|TestModel|TestNetwork|TestOpenAI|TestProvider|TestManualApprovalEnvironmentInActivationJob|TestNeutralToolsIntegration|TestIndividualGitHubToken|TestTopLevelGitHubTokenPrecedence" + - name: "Workflow String & Sanitization" + packages: "./pkg/workflow" + pattern: "String|Sanitize|Normalize|Trim|Clean|Format|SingleQuoteEscaping|WorkflowTimestampCheckUsesJavaScript" + - name: "Workflow Runtime & Setup" + packages: "./pkg/workflow" + pattern: "Runtime|Setup|Install|Download|Version|Binary|StopTimeResolution" + - name: "Workflow Misc Part 2" # Remaining workflow tests + packages: "./pkg/workflow" + pattern: "" + skip_pattern: "TestCompile|TestWorkflow|TestGenerate|TestParse|TestMCP|TestTool|TestSkill|TestPlaywright|TestFirewall|TestValidat|TestLock|TestError|TestWarning|TestRepositoryFeaturesValidation|TestBackwardCompatibilityWithClaudeFormat|SafeOutputs|SafeInputs|CreatePullRequest|OutputLabel|HasSafeOutputs|ConsolidatedSafeOutputs|GitHub|Git|PushToPullRequest|BuildFromAllowed|AdditionalClaudeTools|CopilotGitCommands|Render|Bundle|Script|WritePromptText|^TestCache|TestCacheDependencies|TestCacheKey|TestValidateCache|^TestActionPinSHAsMatchVersionTags|^TestAction[^P]|Container|Dependabot|Security|PII|TestPermissions|TestPackageExtractor|TestCollectPackagesFromWorkflow|TestExtractNpx|TestAgent|TestCopilot|TestCustomEngine|TestEngine|TestModel|TestNetwork|TestOpenAI|TestProvider|TestManualApprovalEnvironmentInActivationJob|TestNeutralToolsIntegration|TestIndividualGitHubToken|TestTopLevelGitHubTokenPrecedence|String|Sanitize|Normalize|Trim|Clean|Format|SingleQuoteEscaping|WorkflowTimestampCheckUsesJavaScript|Runtime|Setup|Install|Download|Version|Binary|StopTimeResolution" + concurrency: + group: ci-${{ github.ref }}-integration-${{ matrix.test-group.name }} + cancel-in-progress: true + name: "Integration: ${{ matrix.test-group.name }}" + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6 + with: + go-version-file: go.mod + cache: true + + - name: Report Go cache status + run: | + if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then + echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY + fi + + - name: Verify dependencies + run: go mod verify + + - name: Run integration tests - ${{ matrix.test-group.name }} + run: | + set -o pipefail + # Sanitize the test group name for use in filename + SAFE_NAME=$(echo "${{ matrix.test-group.name }}" | sed 's/[^a-zA-Z0-9]/-/g' | sed 's/--*/-/g') + + if [ -z "${{ matrix.test-group.pattern }}" ]; then + # Catch-all group: run with -skip to exclude tests matched by other groups + if [ -n "${{ matrix.test-group.skip_pattern || '' }}" ]; then + go test -v -parallel=8 -timeout=5m -tags 'integration' -skip '${{ matrix.test-group.skip_pattern }}' -json ${{ matrix.test-group.packages }} | tee "test-result-integration-${SAFE_NAME}.json" + else + go test -v -parallel=8 -timeout=5m -tags 'integration' -json ${{ matrix.test-group.packages }} | tee "test-result-integration-${SAFE_NAME}.json" + fi + else + go test -v -parallel=8 -timeout=5m -tags 'integration' -run '${{ matrix.test-group.pattern }}' -json ${{ matrix.test-group.packages }} | tee "test-result-integration-${SAFE_NAME}.json" + fi + + - name: Upload integration test results + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: test-result-integration-${{ matrix.test-group.name }} + path: test-result-integration-*.json + retention-days: 14 + + canary_go: + runs-on: ubuntu-latest + needs: [test, integration] + if: always() # Run even if some tests fail to report coverage + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: List all tests in codebase + run: | + set -euo pipefail + echo "Extracting all test function names from source files..." + ./scripts/list-all-tests.sh > all-tests.txt + echo "Found $(wc -l < all-tests.txt) tests in codebase" + + - name: Download all test result artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + path: test-results + pattern: test-result-* + merge-multiple: false + + - name: List downloaded artifacts + run: | + set -euo pipefail + echo "Downloaded test result artifacts:" + find test-results -type f -name "*.json" | sort + echo "" + echo "Total JSON files: $(find test-results -type f -name "*.json" | wc -l)" + + - name: Extract executed tests from artifacts + run: | + set -euo pipefail + echo "Extracting test names from JSON artifacts..." + ./scripts/extract-executed-tests.sh test-results > executed-tests.txt + echo "Found $(wc -l < executed-tests.txt) executed tests" + + - name: Compare test coverage + run: | + ./scripts/compare-test-coverage.sh all-tests.txt executed-tests.txt + + - name: Upload test coverage report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: test-coverage-analysis + path: | + all-tests.txt + executed-tests.txt + retention-days: 14 + + update: + runs-on: ubuntu-latest + permissions: + contents: read + concurrency: + group: ci-${{ github.ref }}-update + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6 + with: + go-version-file: go.mod + cache: true + + - name: Report Go cache status + run: | + if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then + echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY + fi + + - name: Verify dependencies + run: go mod verify + + - name: Build gh-aw binary + run: make build + + - name: Test update command (dry-run) + env: + GH_TOKEN: ${{ github.token }} + run: | + echo "Testing update command to ensure it runs without issues..." + # Run update with verbose flag to check for updates without making changes + # The command checks for gh-aw updates, action updates, and workflow updates + ./gh-aw update --verbose --no-actions + echo "✅ Update command executed successfully" >> $GITHUB_STEP_SUMMARY + + build: + runs-on: ubuntu-latest + permissions: + contents: read + concurrency: + group: ci-${{ github.ref }}-build + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Set up Node.js + id: setup-node + uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6 + with: + node-version: "24" + cache: npm + cache-dependency-path: actions/setup/js/package-lock.json + - name: Report Node cache status + run: | + if [ "${{ steps.setup-node.outputs.cache-hit }}" == "true" ]; then + echo "✅ Node cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Node cache miss" >> $GITHUB_STEP_SUMMARY + fi + - name: Set up Go + id: setup-go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6 + with: + go-version-file: go.mod + cache: true + - name: Report Go cache status + run: | + if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then + echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY + fi + - name: npm ci + run: npm ci + working-directory: ./actions/setup/js + - name: Build code + run: make build + + - name: Upload Linux binary + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: gh-aw-linux-amd64 + path: gh-aw + retention-days: 14 + + - name: Rebuild lock files + run: make recompile + env: + GH_TOKEN: ${{ github.token }} + + validate-yaml: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Check for ANSI escape sequences in YAML files + run: | + echo "🔍 Scanning YAML workflow files for ANSI escape sequences..." + + # Find all YAML files in .github/workflows directory + YAML_FILES=$(find .github/workflows -type f \( -name "*.yml" -o -name "*.yaml" \) | sort) + + # Track if any ANSI codes are found + FOUND_ANSI=0 + + # Check each file for ANSI escape sequences + for file in $YAML_FILES; do + # Use grep to find ANSI escape sequences (ESC [ ... letter) + # The pattern matches: \x1b followed by [ followed by optional digits/semicolons followed by a letter + if grep -P '\x1b\[[0-9;]*[a-zA-Z]' "$file" > /dev/null 2>&1; then + echo "❌ ERROR: Found ANSI escape sequences in: $file" + echo "" + echo "Lines with ANSI codes:" + grep -n -P '\x1b\[[0-9;]*[a-zA-Z]' "$file" || true + echo "" + FOUND_ANSI=1 + fi + done + + if [ $FOUND_ANSI -eq 1 ]; then + echo "" + echo "💡 ANSI escape sequences detected in YAML files!" + echo "" + echo "These are terminal color codes that break YAML parsing." + echo "Common causes:" + echo " - Copy-pasting from colored terminal output" + echo " - Text editors preserving ANSI codes" + echo " - Scripts generating colored output" + echo "" + echo "To fix:" + echo " 1. Remove the ANSI codes from the affected files" + echo " 2. Run 'make recompile' to regenerate workflow files" + echo " 3. Use '--no-color' flags when capturing command output" + echo "" + exit 1 + fi + + echo "✅ No ANSI escape sequences found in YAML files" + + js: + runs-on: ubuntu-latest + needs: validate-yaml + permissions: + contents: read + concurrency: + group: ci-${{ github.ref }}-js + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Set up Node.js + id: setup-node + uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6 + with: + node-version: "24" + cache: npm + cache-dependency-path: actions/setup/js/package-lock.json + - name: Report Node cache status + run: | + if [ "${{ steps.setup-node.outputs.cache-hit }}" == "true" ]; then + echo "✅ Node cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Node cache miss" >> $GITHUB_STEP_SUMMARY + fi + - name: Install npm dependencies + run: cd actions/setup/js && npm ci + - name: Setup prompt templates for tests + run: | + mkdir -p /opt/gh-aw/prompts + cp actions/setup/md/*.md /opt/gh-aw/prompts/ + - name: Run tests + run: cd actions/setup/js && npm test + bench: + # Only run benchmarks on main branch for performance tracking + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: read + concurrency: + group: ci-${{ github.ref }}-bench + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6 + with: + go-version-file: go.mod + cache: true + + - name: Report Go cache status + run: | + if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then + echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY + fi + + - name: Verify dependencies + run: go mod verify + + - name: Run benchmarks + run: make bench + + - name: Display benchmark summary + run: | + echo "## 📊 Benchmark Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + # Show compiler benchmarks from the results + grep "BenchmarkCompile" bench_results.txt | head -20 >> $GITHUB_STEP_SUMMARY || echo "No benchmark results found" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "📁 Full results saved to artifact: benchmark-results" >> $GITHUB_STEP_SUMMARY + + # Benchmark results for performance trend analysis - 14 days allows comparison across multiple runs + - name: Save benchmark results + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: benchmark-results + path: bench_results.txt + if-no-files-found: ignore + retention-days: 14 + + lint-go: + runs-on: ubuntu-latest + permissions: + contents: read + concurrency: + group: ci-${{ github.ref }}-lint-go + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + fetch-depth: 0 # Fetch all history for incremental linting + + - name: Set up Go + id: setup-go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6 + with: + go-version-file: go.mod + cache: true + + - name: Report Go cache status + run: | + if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then + echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY + fi + + # Go formatting check (fast, no deps needed) + - name: Check Go formatting + run: | + unformatted=$(go fmt ./...) + if [ -n "$unformatted" ]; then + echo "❌ Code is not formatted. Run 'make fmt' to fix." >> $GITHUB_STEP_SUMMARY + echo "Unformatted files:" >> $GITHUB_STEP_SUMMARY + echo "$unformatted" >> $GITHUB_STEP_SUMMARY + echo "" + echo "To fix this locally, run:" + echo " make fmt" + echo "" + echo "Or format individual files with:" + echo " go fmt ./path/to/file.go" + exit 1 + fi + echo "✅ Go formatting check passed" >> $GITHUB_STEP_SUMMARY + + # Install only golangci-lint (the only tool needed for linting) + # Other tools (actionlint, gosec, gopls, govulncheck) are not used in this job + - name: Install golangci-lint + run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.7.2 + + # Run golangci-lint via Makefile for consistency + # Uses incremental linting on PRs for faster CI (50-75% speedup) + # Performance optimizations in .golangci.yml: + # - timeout: 5m prevents hanging + # - modules-download-mode: readonly uses cached modules only + - name: Run golangci-lint + run: | + export PATH="$PATH:$(go env GOPATH)/bin" + if [ "${{ github.event_name }}" = "pull_request" ]; then + # Incremental linting on PRs - only check changed files + # This provides 50-75% faster linting on typical PRs + BASE_REF="origin/${{ github.base_ref }}" + if git rev-parse --verify "$BASE_REF" >/dev/null 2>&1; then + echo "Using incremental lint against $BASE_REF" + make golint-incremental BASE_REF="$BASE_REF" + else + echo "⚠️ Base ref $BASE_REF not found, falling back to full lint" + make golint + fi + else + # Full scan on main branch to ensure comprehensive coverage + make golint + fi + + # Error message linting (requires Go only) + - name: Lint error messages + run: make lint-errors + + lint-js: + runs-on: ubuntu-latest + permissions: + contents: read + concurrency: + group: ci-${{ github.ref }}-lint-js + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Node.js + id: setup-node + uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6 + with: + node-version: "24" + cache: npm + cache-dependency-path: actions/setup/js/package-lock.json + + - name: Report Node cache status + run: | + if [ "${{ steps.setup-node.outputs.cache-hit }}" == "true" ]; then + echo "✅ Node cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Node cache miss" >> $GITHUB_STEP_SUMMARY + fi + + - name: Install npm dependencies + run: cd actions/setup/js && npm ci + + # JavaScript and JSON formatting checks + - name: Lint JavaScript files + run: make lint-cjs + + - name: Check JSON formatting + run: make fmt-check-json + + audit: + runs-on: ubuntu-latest + permissions: + contents: read + concurrency: + group: ci-${{ github.ref }}-audit + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6 + with: + go-version-file: go.mod + cache: true + + - name: Report Go cache status + run: | + if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then + echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY + fi + + - name: Verify dependencies + run: go mod verify + + - name: Build gh-aw binary + run: make build + + - name: Run dependency audit (human-readable) + run: | + echo "## Dependency Health Audit" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + ./gh-aw update --audit 2>&1 | tee audit_output.txt || true + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + head -100 audit_output.txt >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + + - name: Run dependency audit (JSON) + id: audit_json + run: | + # Run audit with JSON output for agent-friendly parsing + ./gh-aw update --audit --json > audit.json 2>&1 + + # Display summary in GitHub Actions + echo "✅ Dependency audit completed" >> $GITHUB_STEP_SUMMARY + + # Extract key metrics + TOTAL_DEPS=$(jq '.summary.total_dependencies' audit.json) + OUTDATED=$(jq '.summary.outdated_count' audit.json) + SECURITY=$(jq '.summary.security_advisories' audit.json) + V0_PERCENT=$(jq '.summary.v0_percentage' audit.json) + + echo "📊 **Audit Results:**" >> $GITHUB_STEP_SUMMARY + echo "- Total dependencies: $TOTAL_DEPS" >> $GITHUB_STEP_SUMMARY + echo "- Outdated: $OUTDATED" >> $GITHUB_STEP_SUMMARY + echo "- Security advisories: $SECURITY" >> $GITHUB_STEP_SUMMARY + echo "- v0.x exposure: ${V0_PERCENT}%" >> $GITHUB_STEP_SUMMARY + + - name: Upload audit results + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: dependency-audit + path: | + audit.json + audit_output.txt + retention-days: 30 + + actions-build: + runs-on: ubuntu-latest + permissions: + contents: read + concurrency: + group: ci-${{ github.ref }}-actions-build + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6 + with: + go-version-file: go.mod + cache: true + + - name: Report Go cache status + run: | + if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then + echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY + fi + + - name: Verify dependencies + run: go mod verify + + - name: Build actions + run: make actions-build + + - name: Validate actions + run: make actions-validate + + fuzz: + # Only run fuzz tests on main branch (10s is insufficient for PRs) + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: read + concurrency: + group: ci-${{ github.ref }}-fuzz + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6 + with: + go-version-file: go.mod + cache: true + + - name: Report Go cache status + run: | + if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then + echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY + fi + + - name: Verify dependencies + run: go mod verify + + - name: Run fuzz tests + run: | + # Create directory for fuzz results + mkdir -p fuzz-results + + # Run fuzz tests and capture output + go test -run='^$' -fuzz=FuzzParseFrontmatter -fuzztime=10s ./pkg/parser/ 2>&1 | tee fuzz-results/FuzzParseFrontmatter.txt + go test -run='^$' -fuzz=FuzzScheduleParser -fuzztime=10s ./pkg/parser/ 2>&1 | tee fuzz-results/FuzzScheduleParser.txt + go test -run='^$' -fuzz=FuzzExpressionParser -fuzztime=10s ./pkg/workflow/ 2>&1 | tee fuzz-results/FuzzExpressionParser.txt + go test -run='^$' -fuzz=FuzzMentionsFiltering -fuzztime=10s ./pkg/workflow/ 2>&1 | tee fuzz-results/FuzzMentionsFiltering.txt + go test -run='^$' -fuzz=FuzzSanitizeOutput -fuzztime=10s ./pkg/workflow/ 2>&1 | tee fuzz-results/FuzzSanitizeOutput.txt + go test -run='^$' -fuzz=FuzzSanitizeIncomingText -fuzztime=10s ./pkg/workflow/ 2>&1 | tee fuzz-results/FuzzSanitizeIncomingText.txt + go test -run='^$' -fuzz=FuzzSanitizeLabelContent -fuzztime=10s ./pkg/workflow/ 2>&1 | tee fuzz-results/FuzzSanitizeLabelContent.txt + go test -run='^$' -fuzz=FuzzWrapExpressionsInTemplateConditionals -fuzztime=10s ./pkg/workflow/ 2>&1 | tee fuzz-results/FuzzWrapExpressionsInTemplateConditionals.txt + go test -run='^$' -fuzz=FuzzYAMLParsing -fuzztime=10s ./pkg/workflow/ 2>&1 | tee fuzz-results/FuzzYAMLParsing.txt + go test -run='^$' -fuzz=FuzzTemplateRendering -fuzztime=10s ./pkg/workflow/ 2>&1 | tee fuzz-results/FuzzTemplateRendering.txt + go test -run='^$' -fuzz=FuzzInputValidation -fuzztime=10s ./pkg/workflow/ 2>&1 | tee fuzz-results/FuzzInputValidation.txt + go test -run='^$' -fuzz=FuzzNetworkPermissions -fuzztime=10s ./pkg/workflow/ 2>&1 | tee fuzz-results/FuzzNetworkPermissions.txt + go test -run='^$' -fuzz=FuzzSafeJobConfig -fuzztime=10s ./pkg/workflow/ 2>&1 | tee fuzz-results/FuzzSafeJobConfig.txt + + # Copy fuzz corpus data (testdata/fuzz directories) + echo "Copying fuzz corpus data..." + find ./pkg -path "*/testdata/fuzz" -type d | while read -r dir; do + pkg_name=$(echo "$dir" | sed 's|^\./pkg/||' | sed 's|/testdata/fuzz$||') + echo "Copying corpus from $dir to fuzz-results/corpus/$pkg_name/" + mkdir -p "fuzz-results/corpus/$pkg_name" + cp -r "$dir"/* "fuzz-results/corpus/$pkg_name/" 2>/dev/null || echo "No corpus data in $dir" + done + + - name: Upload fuzz test results + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: fuzz-results + path: fuzz-results/ + retention-days: 14 + + security: + runs-on: ubuntu-latest + permissions: + contents: read + concurrency: + group: ci-${{ github.ref }}-security + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6 + with: + go-version-file: go.mod + cache: true + + - name: Report Go cache status + run: | + if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then + echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY + fi + + - name: Verify dependencies + run: go mod verify + + - name: Run security regression tests + run: make test-security + + security-scan: + # Only run security scans on main branch to reduce PR overhead + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 10 # Prevent jobs from hanging indefinitely + permissions: + contents: read + strategy: + fail-fast: false + matrix: + tool: + - name: zizmor + flag: --zizmor + - name: actionlint + flag: --actionlint + - name: poutine + flag: --poutine + concurrency: + group: ci-${{ github.ref }}-security-scan-${{ matrix.tool.name }} + cancel-in-progress: true + name: "Security Scan: ${{ matrix.tool.name }}" + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6 + with: + go-version-file: go.mod + cache: true + + - name: Report Go cache status + run: | + if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then + echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY + fi + + - name: Build gh-aw + run: make build + + - name: Run ${{ matrix.tool.name }} security scan on poem workflow + run: ./gh-aw compile poem-bot ${{ matrix.tool.flag }} --verbose + + logs-token-check: + if: false + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + concurrency: + group: ci-${{ github.ref }}-logs-token-check + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6 + with: + go-version-file: go.mod + cache: true + + - name: Build gh-aw + run: make build + + - name: Run logs command with JSON output + id: logs_check + run: | + set -e # Fail on first error + # Run the logs command and capture only stdout (JSON output) + # stderr is not redirected, so warning messages go to console + ./gh-aw logs smoke-copilot -c 2 --json --verbose > logs_output.json + + # Display the output for debugging + echo "Logs command output:" + cat logs_output.json + + # Check if the JSON structure is valid + echo "## Validating JSON Structure" + + # Check if token count is found in the JSON output + if jq -e '.summary.total_tokens' logs_output.json > /dev/null 2>&1; then + TOKEN_COUNT=$(jq '.summary.total_tokens' logs_output.json) + echo "✅ Token count found: $TOKEN_COUNT" + + # Validate that token count is greater than 0 + if [ "$TOKEN_COUNT" -gt 0 ]; then + echo "✅ Token count is greater than 0: $TOKEN_COUNT" + echo "token_count=$TOKEN_COUNT" >> $GITHUB_OUTPUT + else + echo "❌ Token count is 0 - expected tokens to be parsed from logs" + exit 1 + fi + else + echo "❌ Token count not found in JSON output" + exit 1 + fi + + # Check if runs array exists (even if empty) + if jq -e '.runs' logs_output.json > /dev/null 2>&1; then + RUNS_COUNT=$(jq '.runs | length' logs_output.json) + echo "✅ Runs array found: $RUNS_COUNT runs" + else + echo "❌ Runs array not found in JSON output" + exit 1 + fi + + # If there are runs, validate that key fields are resolved + if [ "$RUNS_COUNT" -gt 0 ]; then + # Check if agent (engine_id) field exists in first run + if jq -e '.runs[0] | has("agent")' logs_output.json > /dev/null 2>&1; then + AGENT=$(jq -r '.runs[0].agent // "null"' logs_output.json) + echo "✅ Agent field found in run: $AGENT" + else + echo "❌ Agent field not found in run data" + exit 1 + fi + + # Check if workflow_path field exists in first run + if jq -e '.runs[0] | has("workflow_path")' logs_output.json > /dev/null 2>&1; then + WORKFLOW_PATH=$(jq -r '.runs[0].workflow_path // "null"' logs_output.json) + echo "✅ Workflow path field found in run: $WORKFLOW_PATH" + else + echo "❌ Workflow path field not found in run data" + exit 1 + fi + + # Check if workflow_name is present + if jq -e '.runs[0].workflow_name' logs_output.json > /dev/null 2>&1; then + WORKFLOW_NAME=$(jq -r '.runs[0].workflow_name' logs_output.json) + echo "✅ Workflow name found in run: $WORKFLOW_NAME" + else + echo "❌ Workflow name not found in run data" + exit 1 + fi + else + echo "ℹ️ No runs found to validate (this is ok)" + fi + + echo "✅ All JSON structure validations passed" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + mcp-server-compile-test: + runs-on: ubuntu-latest + permissions: + contents: read + concurrency: + group: ci-${{ github.ref }}-mcp-server-compile-test + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6 + with: + go-version-file: go.mod + cache: true + + - name: Report Go cache status + run: | + if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then + echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY + fi + + - name: Build gh-aw binary + run: make build + + - name: Create test workflow with error + run: | + mkdir -p .github/workflows + cat > .github/workflows/test-invalid.md << 'EOF' + --- + on: push + engine: copilot + invalid_field: this will cause an error + --- + # Test Invalid Workflow + + This workflow has an invalid field that will cause a compilation error. + EOF + + - name: Test MCP server compile tool + run: | + # Create a test script using the MCP Go SDK + cat > test_mcp_compile.go << 'GOEOF' + package main + + import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "time" + + "github.com/modelcontextprotocol/go-sdk/mcp" + ) + + func main() { + // Create MCP client + client := mcp.NewClient(&mcp.Implementation{ + Name: "ci-test-client", + Version: "1.0.0", + }, nil) + + // Start the MCP server as a subprocess with absolute path + binaryPath := "./gh-aw" + serverCmd := exec.Command(binaryPath, "mcp-server", "--cmd", binaryPath) + transport := &mcp.CommandTransport{Command: serverCmd} + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Connect to the server + session, err := client.Connect(ctx, transport, nil) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to connect to MCP server: %v\n", err) + os.Exit(1) + } + defer session.Close() + + fmt.Println("✅ Successfully connected to MCP server") + + // Call the compile tool with the invalid workflow + params := &mcp.CallToolParams{ + Name: "compile", + Arguments: map[string]any{ + "workflows": []string{"test-invalid.md"}, + }, + } + + result, err := session.CallTool(ctx, params) + if err != nil { + fmt.Fprintf(os.Stderr, "MCP tool call returned error (this is expected): %v\n", err) + // Check if the error contains expected error information + fmt.Println("✅ Compile tool correctly returned an error") + os.Exit(0) + } + + // Get the result content + if len(result.Content) == 0 { + fmt.Fprintln(os.Stderr, "❌ Expected non-empty result from compile tool") + os.Exit(1) + } + + textContent, ok := result.Content[0].(*mcp.TextContent) + if !ok { + fmt.Fprintln(os.Stderr, "❌ Expected text content from compile tool") + os.Exit(1) + } + + fmt.Printf("Compile tool output:\n%s\n", textContent.Text) + + // Parse the JSON output to check for errors + var compileResults []map[string]any + if err := json.Unmarshal([]byte(textContent.Text), &compileResults); err != nil { + fmt.Fprintf(os.Stderr, "Failed to parse JSON output: %v\n", err) + os.Exit(1) + } + + // Check if the workflow is marked as invalid + if len(compileResults) == 0 { + fmt.Fprintln(os.Stderr, "❌ Expected at least one workflow result") + os.Exit(1) + } + + result0 := compileResults[0] + valid, ok := result0["valid"].(bool) + if !ok { + fmt.Fprintln(os.Stderr, "❌ Expected 'valid' field in result") + os.Exit(1) + } + + if valid { + fmt.Fprintln(os.Stderr, "❌ Expected workflow to be invalid") + os.Exit(1) + } + + // Check that errors field exists and has at least one error + errors, ok := result0["errors"].([]any) + if !ok || len(errors) == 0 { + fmt.Fprintln(os.Stderr, "❌ Expected errors array with at least one error") + os.Exit(1) + } + + fmt.Println("✅ Compile tool correctly reported validation errors:") + errorsJSON, _ := json.MarshalIndent(errors, " ", " ") + fmt.Printf(" %s\n", string(errorsJSON)) + + os.Exit(0) + } + GOEOF + + # Run the test + go run test_mcp_compile.go + + - name: Report test results + if: always() + run: | + echo "## MCP Server Compile Tool Test" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "This test verifies that:" >> $GITHUB_STEP_SUMMARY + echo "1. The gh-aw MCP server can be started successfully" >> $GITHUB_STEP_SUMMARY + echo "2. The compile tool can be invoked through the MCP server" >> $GITHUB_STEP_SUMMARY + echo "3. The compile tool correctly detects and reports validation errors" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "✅ Test completed successfully" >> $GITHUB_STEP_SUMMARY + + cross-platform-build: + name: Build & Test on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + permissions: + contents: read + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - windows-latest + concurrency: + group: ci-${{ github.ref }}-cross-platform-${{ matrix.os }} + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6 + with: + go-version-file: go.mod + cache: true + + - name: Report Go cache status + shell: bash + run: | + if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then + echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY + fi + + - name: Verify dependencies + run: go mod verify + + - name: Build gh-aw binary + run: make build + + - name: Create test workflow + shell: bash + run: | + mkdir -p .github/workflows + cat > .github/workflows/test-cross-platform.md << 'EOF' + --- + on: push + engine: copilot + --- + # Test Workflow for Cross-Platform CI + + This is a simple test workflow to verify the compile command works correctly. + + ## Task + Echo hello world. + EOF + + - name: Test compile command + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + echo "Testing compile command on ${{ matrix.os }}..." + + # Determine binary name based on OS + if [[ "$RUNNER_OS" == "Windows" ]]; then + BINARY="./gh-aw.exe" + else + BINARY="./gh-aw" + fi + + # Verify binary exists + if [ ! -f "$BINARY" ]; then + echo "❌ Binary not found: $BINARY" + ls -la + exit 1 + fi + + # Run compile command + "$BINARY" compile test-cross-platform --verbose + + # Check if lock file was generated + if [ -f ".github/workflows/test-cross-platform.lock.yml" ]; then + echo "✅ Compile succeeded - lock file generated" + else + echo "❌ Compile failed - no lock file generated" + exit 1 + fi + + echo "## Cross-Platform Test Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "✅ Successfully compiled workflow on ${{ matrix.os }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Platform:** ${{ matrix.os }}" >> $GITHUB_STEP_SUMMARY + echo "**Binary:** $BINARY" >> $GITHUB_STEP_SUMMARY + echo "**Go version:** $(go version)" >> $GITHUB_STEP_SUMMARY + + - name: Clean up test files + if: always() + shell: bash + run: | + rm -f .github/workflows/test-cross-platform.md + rm -f .github/workflows/test-cross-platform.lock.yml + + alpine-container-test: + name: Alpine Container Test + runs-on: ubuntu-latest + permissions: + contents: read + concurrency: + group: ci-${{ github.ref }}-alpine-container + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6 + with: + go-version-file: go.mod + cache: true + + - name: Report Go cache status + run: | + if [ "${{ steps.setup-go.outputs.cache-hit }}" == "true" ]; then + echo "✅ Go cache hit" >> $GITHUB_STEP_SUMMARY + else + echo "⚠️ Go cache miss" >> $GITHUB_STEP_SUMMARY + fi + + - name: Verify dependencies + run: go mod verify + + - name: Build Linux binary for Alpine + run: make build-linux + + - name: Build Alpine Docker image + run: | + echo "Building Alpine Docker image..." + docker build -t gh-aw-alpine:test \ + --build-arg BINARY=gh-aw-linux-amd64 \ + -f Dockerfile . + echo "✅ Alpine Docker image built successfully" + + - name: Test Docker image basic commands + run: | + echo "Testing Docker image basic commands..." + docker run --rm gh-aw-alpine:test --version + docker run --rm gh-aw-alpine:test --help + echo "✅ Basic commands work" + + - name: Create test workflow in container + run: | + echo "Creating test workflow file..." + mkdir -p test-workspace/.github/workflows + cat > test-workspace/.github/workflows/test-alpine.md << 'EOF' + --- + on: push + engine: copilot + --- + # Test Workflow for Alpine Container + + This is a simple test workflow to verify the compile command works correctly in Alpine container. + + ## Task + Echo hello from Alpine container. + EOF + echo "✅ Test workflow created" + + - name: Run compile through Alpine container + run: | + echo "Running compile command through Alpine container..." + docker run --rm \ + -v "$(pwd)/test-workspace:/workspace" \ + -w /workspace \ + gh-aw-alpine:test compile test-alpine --verbose + + echo "✅ Compile command executed" + + - name: Verify lock file generation + run: | + echo "Verifying lock file was generated..." + if [ -f "test-workspace/.github/workflows/test-alpine.lock.yml" ]; then + echo "✅ Lock file generated successfully" + echo "" + echo "Lock file contents:" + head -20 test-workspace/.github/workflows/test-alpine.lock.yml + else + echo "❌ Lock file not found" + ls -la test-workspace/.github/workflows/ + exit 1 + fi + + - name: Generate test summary + if: always() + run: | + echo "## Alpine Container Test Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "This test verifies that:" >> $GITHUB_STEP_SUMMARY + echo "1. The Alpine Docker image can be built successfully" >> $GITHUB_STEP_SUMMARY + echo "2. The gh-aw binary works correctly in Alpine Linux" >> $GITHUB_STEP_SUMMARY + echo "3. The compile command can process workflows in the container" >> $GITHUB_STEP_SUMMARY + echo "4. Lock files are generated correctly" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + if [ -f "test-workspace/.github/workflows/test-alpine.lock.yml" ]; then + echo "✅ All tests passed successfully" >> $GITHUB_STEP_SUMMARY + else + echo "❌ Lock file generation failed" >> $GITHUB_STEP_SUMMARY + fi + + - name: Clean up test files + if: always() + run: | + rm -rf test-workspace diff --git a/.github/workflows/ci-coach.md b/.github/workflows/ci-coach.md index 020958cc9a..c0aadd0bfc 100644 --- a/.github/workflows/ci-coach.md +++ b/.github/workflows/ci-coach.md @@ -37,7 +37,7 @@ Analyze the CI workflow daily to identify concrete optimization opportunities th - **Repository**: ${{ github.repository }} - **Run Number**: #${{ github.run_number }} -- **Target Workflow**: `.github/workflows/ci.yml` +- **Target Workflow**: `.github/workflows/agent-ci.yml` ## Data Available @@ -48,7 +48,7 @@ The `ci-data-analysis` shared module has pre-downloaded CI run data and built th 1. **CI Runs**: `/tmp/ci-runs.json` - Last 100 workflow runs 2. **Artifacts**: `/tmp/ci-artifacts/` - Coverage reports, benchmarks, and **fuzz test results** -3. **CI Configuration**: `.github/workflows/ci.yml` - Current workflow +3. **CI Configuration**: `.github/workflows/agent-ci.yml` - Current workflow 4. **Cache Memory**: `/tmp/cache-memory/` - Historical analysis data 5. **Test Results**: `/tmp/gh-aw/test-results.json` - Test performance data 6. **Fuzz Results**: `/tmp/ci-artifacts/*/fuzz-results/` - Fuzz test output and corpus data @@ -106,7 +106,7 @@ If you identify improvements worth implementing: If you identify improvements worth implementing: -1. **Make focused changes** to `.github/workflows/ci.yml`: +1. **Make focused changes** to `.github/workflows/agent-ci.yml`: - Use the `edit` tool to make precise modifications - Keep changes minimal and well-documented - Add comments explaining why changes improve efficiency diff --git a/scripts/compare-test-coverage.sh b/scripts/compare-test-coverage.sh new file mode 100755 index 0000000000..cf368c0980 --- /dev/null +++ b/scripts/compare-test-coverage.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Compare all tests vs executed tests to find any missing test coverage +# This ensures all tests are being run in CI unless explicitly skipped + +set -euo pipefail + +if [ $# -ne 2 ]; then + echo "Usage: $0 " + echo "Compares the two lists and reports any missing tests" + exit 1 +fi + +ALL_TESTS_FILE="$1" +EXECUTED_TESTS_FILE="$2" + +if [ ! -f "$ALL_TESTS_FILE" ]; then + echo "Error: All tests file $ALL_TESTS_FILE does not exist" + exit 1 +fi + +if [ ! -f "$EXECUTED_TESTS_FILE" ]; then + echo "Error: Executed tests file $EXECUTED_TESTS_FILE does not exist" + exit 1 +fi + +echo "📊 Test Coverage Analysis" +echo "=========================" +echo "" + +ALL_COUNT=$(wc -l < "$ALL_TESTS_FILE") +EXECUTED_COUNT=$(wc -l < "$EXECUTED_TESTS_FILE") + +echo "Total tests defined in codebase: $ALL_COUNT" +echo "Tests executed in CI: $EXECUTED_COUNT" +echo "" + +# Find tests that are defined but not executed +MISSING_TESTS=$(comm -23 "$ALL_TESTS_FILE" "$EXECUTED_TESTS_FILE") + +if [ -z "$MISSING_TESTS" ]; then + echo "✅ SUCCESS: All tests are being executed in CI!" + echo "" + echo "Test coverage: 100% ($EXECUTED_COUNT/$ALL_COUNT tests executed)" + exit 0 +else + MISSING_COUNT=$(echo "$MISSING_TESTS" | wc -l) + COVERAGE_PERCENT=$(awk "BEGIN {printf \"%.1f\", ($EXECUTED_COUNT / $ALL_COUNT) * 100}") + + echo "❌ FAILURE: Found $MISSING_COUNT tests that are NOT being executed in CI" + echo "" + echo "Test coverage: $COVERAGE_PERCENT% ($EXECUTED_COUNT/$ALL_COUNT tests executed)" + echo "" + echo "Missing tests:" + echo "==============" + echo "$MISSING_TESTS" | head -20 + + if [ "$MISSING_COUNT" -gt 20 ]; then + echo "... and $((MISSING_COUNT - 20)) more" + fi + + echo "" + echo "These tests are defined in *_test.go files but were not executed" + echo "in any of the test jobs. Please either:" + echo " 1. Add them to the appropriate test job pattern, or" + echo " 2. Remove them if they are obsolete" + + exit 1 +fi diff --git a/scripts/extract-executed-tests.sh b/scripts/extract-executed-tests.sh new file mode 100755 index 0000000000..a71d387e69 --- /dev/null +++ b/scripts/extract-executed-tests.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Extract executed test names from JSON test result files +# Parses the JSON output from 'go test -json' format + +set -euo pipefail + +if [ $# -eq 0 ]; then + echo "Usage: $0 " + echo "Extracts executed test names from JSON test result files in the specified directory" + exit 1 +fi + +TEST_RESULT_DIR="$1" + +if [ ! -d "$TEST_RESULT_DIR" ]; then + echo "Error: Directory $TEST_RESULT_DIR does not exist" + exit 1 +fi + +# Find all JSON test result files and extract test names +# Look for lines with "Action":"run" and extract the "Test" field +# Process each file separately to handle cases where files might be empty or have no matches +temp_file=$(mktemp) +find "$TEST_RESULT_DIR" -name "*.json" -type f | while read -r file; do + if [ -s "$file" ]; then + # File exists and is not empty + grep '"Action":"run"' "$file" 2>/dev/null | \ + grep -o '"Test":"[^"]*"' | \ + sed 's/"Test":"\([^"]*\)"/\1/' >> "$temp_file" || true + fi +done + +# Sort and deduplicate the results +if [ -s "$temp_file" ]; then + sort -u "$temp_file" + rm -f "$temp_file" +else + # No tests found - this is an error condition + rm -f "$temp_file" + echo "Error: No test execution records found in $TEST_RESULT_DIR" >&2 + exit 1 +fi diff --git a/scripts/list-all-tests.sh b/scripts/list-all-tests.sh new file mode 100755 index 0000000000..d6617e58f9 --- /dev/null +++ b/scripts/list-all-tests.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# List all Go test function names from source files +# This script extracts test function names from *_test.go files + +set -euo pipefail + +# Find all test files and extract test function names +find . -name "*_test.go" -type f -exec grep -h "^func Test" {} \; | \ + sed 's/func \(Test[^(]*\).*/\1/' | \ + sort -u