diff --git a/.github/workflows/autobahn.yml b/.github/workflows/autobahn.yml index e3c05dc5ea1..9117497c574 100644 --- a/.github/workflows/autobahn.yml +++ b/.github/workflows/autobahn.yml @@ -47,5 +47,24 @@ jobs: run: npm run test:websocket:autobahn env: FUZZING_SERVER_URL: ws://fuzzingserver:9001 - - name: Report + + - name: Report into CI + id: report-ci run: npm run test:websocket:autobahn:report + + - name: Generate Report for PR Comment + if: github.event_name == 'pull_request' + id: report-markdown + run: | + echo "comment<> $GITHUB_OUTPUT + node test/autobahn/report.js >> $GITHUB_OUTPUT + echo "nEOFn" >> $GITHUB_OUTPUT + env: + REPORTER: markdown + + - name: Comment PR + if: github.event_name == 'pull_request' + uses: thollander/actions-comment-pull-request@v2 + with: + message: ${{ steps.report-markdown.outputs.comment }} + comment_tag: autobahn diff --git a/test/autobahn/report.js b/test/autobahn/report.js index 3c4f53fb167..105eb4f1496 100644 --- a/test/autobahn/report.js +++ b/test/autobahn/report.js @@ -2,6 +2,21 @@ const result = require('./reports/clients/index.json').undici +const failOnError = process.env.FAIL_ON_ERROR === 'true' +const reporter = process.env.REPORTER || 'table' +let runFailed = false + +let okTests = 0 +let failedTests = 0 +let nonStrictTests = 0 +let wrongCodeTests = 0 +let uncleanTests = 0 +let failedByClientTests = 0 +let informationalTests = 0 +let unimplementedTests = 0 + +let totalTests = 0 + function testCaseIdToWeight (testCaseId) { const [major, minor, sub] = testCaseId.split('.') return sub @@ -9,6 +24,19 @@ function testCaseIdToWeight (testCaseId) { : parseInt(major, 10) * 10000 + parseInt(minor, 10) * 100 } +function isFailedTestCase (testCase) { + return ( + testCase.behavior === 'FAILED' || + testCase.behavior === 'WRONG CODE' || + testCase.behavior === 'UNCLEAN' || + testCase.behavior === 'FAILED BY CLIENT' || + testCase.behaviorClose === 'FAILED' || + testCase.behaviorClose === 'WRONG CODE' || + testCase.behaviorClose === 'UNCLEAN' || + testCase.behaviorClose === 'FAILED BY CLIENT' + ) +} + const keys = Object.keys(result).sort((a, b) => { a = testCaseIdToWeight(a) b = testCaseIdToWeight(b) @@ -18,6 +46,97 @@ const keys = Object.keys(result).sort((a, b) => { const reorderedResult = {} for (const key of keys) { reorderedResult[key] = result[key] + delete reorderedResult[key].reportfile + + totalTests++ + + if ( + failOnError && + !runFailed && + isFailedTestCase(result[key]) + ) { + runFailed = true + } + + switch (result[key].behavior) { + case 'OK': + okTests++ + break + case 'FAILED': + failedTests++ + break + case 'NON-STRICT': + nonStrictTests++ + break + case 'WRONG CODE': + wrongCodeTests++ + break + case 'UNCLEAN': + uncleanTests++ + break + case 'FAILED BY CLIENT': + failedByClientTests++ + break + case 'INFORMATIONAL': + informationalTests++ + break + case 'UNIMPLEMENTED': + unimplementedTests++ + break + } +} + +if ( + reporter === 'table' +) { + console.log('Autobahn Test Report\n\nSummary:') + + console.table({ + OK: okTests, + Failed: failedTests, + 'Non-Strict': nonStrictTests, + 'Wrong Code': wrongCodeTests, + Unclean: uncleanTests, + 'Failed By Client': failedByClientTests, + Informational: informationalTests, + Unimplemented: unimplementedTests, + 'Total Tests': totalTests + }) + + console.log('Details:') + + console.table(reorderedResult) +} + +if (reporter === 'markdown') { + console.log(`## Autobahn Test Report + +### Summary + +| Type | Count | +|---|---| +| OK | ${okTests} | +| Failed | ${failedTests} | +| Non-Strict | ${nonStrictTests} | +| Wrong Code | ${wrongCodeTests} | +| Unclean | ${uncleanTests} | +| Failed By Client | ${failedByClientTests} | +| Informational | ${informationalTests} | +| Unimplemented | ${unimplementedTests} | +| Total Tests | ${totalTests} | + +
+Details + +| Test Case | Behavior | Close Behavior | Duration | Remote Close Code | +|---|---|---|---|---| +${keys.map(key => { + const testCase = reorderedResult[key] + return `| ${key} | ${testCase.behavior} | ${testCase.behaviorClose} | ${testCase.duration} | ${testCase.remoteCloseCode} |` +}).join('\n')} + +
+`) } -console.table(reorderedResult) +process.exit(runFailed ? 1 : 0)