diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml deleted file mode 100644 index 92524ea17..000000000 --- a/.github/workflows/conformance.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Conformance Test - -on: - pull_request: - -permissions: - contents: read - -jobs: - conformance: - runs-on: ubuntu-latest - - steps: - - name: Check out code - uses: actions/checkout@v6 - with: - # Fetch full history to access merge-base - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@v6 - with: - go-version-file: "go.mod" - - - name: Download dependencies - run: go mod download - - - name: Run conformance test - id: conformance - run: | - # Run conformance test, capture stdout for summary - script/conformance-test > conformance-summary.txt 2>&1 || true - - # Output the summary - cat conformance-summary.txt - - # Check result - if grep -q "RESULT: ALL TESTS PASSED" conformance-summary.txt; then - echo "status=passed" >> $GITHUB_OUTPUT - else - echo "status=differences" >> $GITHUB_OUTPUT - fi - - - name: Generate Job Summary - run: | - # Add the full markdown report to the job summary - echo "# MCP Server Conformance Report" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Comparing PR branch against merge-base with \`origin/main\`" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - # Extract and append the report content (skip the header since we added our own) - tail -n +5 conformance-report/CONFORMANCE_REPORT.md >> $GITHUB_STEP_SUMMARY - - echo "" >> $GITHUB_STEP_SUMMARY - echo "---" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - # Add interpretation note - if [ "${{ steps.conformance.outputs.status }}" = "passed" ]; then - echo "✅ **All conformance tests passed** - No behavioral differences detected." >> $GITHUB_STEP_SUMMARY - else - echo "⚠️ **Differences detected** - Review the diffs above to ensure changes are intentional." >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Common expected differences:" >> $GITHUB_STEP_SUMMARY - echo "- New tools/toolsets added" >> $GITHUB_STEP_SUMMARY - echo "- Tool descriptions updated" >> $GITHUB_STEP_SUMMARY - echo "- Capability changes (intentional improvements)" >> $GITHUB_STEP_SUMMARY - fi diff --git a/.github/workflows/mcp-diff.yml b/.github/workflows/mcp-diff.yml new file mode 100644 index 000000000..1e18a48b3 --- /dev/null +++ b/.github/workflows/mcp-diff.yml @@ -0,0 +1,67 @@ +name: MCP Server Diff + +on: + pull_request: + +permissions: + contents: read + +jobs: + mcp-diff: + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Run MCP Server Diff + uses: SamMorrowDrums/mcp-server-diff@v2 + with: + setup_go: "true" + install_command: go mod download + start_command: go run ./cmd/github-mcp-server stdio + env_vars: | + GITHUB_PERSONAL_ACCESS_TOKEN=test-token + configurations: | + [ + {"name": "default", "args": ""}, + {"name": "read-only", "args": "--read-only"}, + {"name": "dynamic-toolsets", "args": "--dynamic-toolsets"}, + {"name": "read-only+dynamic", "args": "--read-only --dynamic-toolsets"}, + {"name": "toolsets-repos", "args": "--toolsets=repos"}, + {"name": "toolsets-issues", "args": "--toolsets=issues"}, + {"name": "toolsets-pull_requests", "args": "--toolsets=pull_requests"}, + {"name": "toolsets-repos,issues", "args": "--toolsets=repos,issues"}, + {"name": "toolsets-all", "args": "--toolsets=all"}, + {"name": "tools-get_me", "args": "--tools=get_me"}, + {"name": "tools-get_me,list_issues", "args": "--tools=get_me,list_issues"}, + {"name": "toolsets-repos+read-only", "args": "--toolsets=repos --read-only"}, + {"name": "toolsets-all+dynamic", "args": "--toolsets=all --dynamic-toolsets"}, + {"name": "toolsets-repos+dynamic", "args": "--toolsets=repos --dynamic-toolsets"}, + {"name": "toolsets-repos,issues+dynamic", "args": "--toolsets=repos,issues --dynamic-toolsets"}, + { + "name": "dynamic-tool-calls", + "args": "--dynamic-toolsets", + "custom_messages": [ + {"id": 10, "name": "list_toolsets_before", "message": {"jsonrpc": "2.0", "id": 10, "method": "tools/call", "params": {"name": "list_available_toolsets", "arguments": {}}}}, + {"id": 11, "name": "get_toolset_tools", "message": {"jsonrpc": "2.0", "id": 11, "method": "tools/call", "params": {"name": "get_toolset_tools", "arguments": {"toolset": "repos"}}}}, + {"id": 12, "name": "enable_toolset", "message": {"jsonrpc": "2.0", "id": 12, "method": "tools/call", "params": {"name": "enable_toolset", "arguments": {"toolset": "repos"}}}}, + {"id": 13, "name": "list_toolsets_after", "message": {"jsonrpc": "2.0", "id": 13, "method": "tools/call", "params": {"name": "list_available_toolsets", "arguments": {}}}} + ] + } + ] + + - name: Add interpretation note + if: always() + run: | + echo "" >> $GITHUB_STEP_SUMMARY + echo "---" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "ℹ️ **Note:** Differences may be intentional improvements." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Common expected differences:" >> $GITHUB_STEP_SUMMARY + echo "- New tools/toolsets added" >> $GITHUB_STEP_SUMMARY + echo "- Tool descriptions updated" >> $GITHUB_STEP_SUMMARY + echo "- Capability changes (intentional improvements)" >> $GITHUB_STEP_SUMMARY