diff --git a/packages/api/lib/testrun.js b/packages/api/lib/testrun.js index bee5e13c..1df0b1b3 100644 --- a/packages/api/lib/testrun.js +++ b/packages/api/lib/testrun.js @@ -15,16 +15,15 @@ const firebaseEncode = require('./firebase-encode'); class TestCaseRun { - constructor (okMessage, number, name) { + constructor (okMessage, name) { this.successful = (okMessage === 'ok'); - this.number = number; this.name = name; this.encodedName = firebaseEncode(this.name); this.failureMessage = 'Successful'; } display () { - return this.number + ', ' + this.name + ', ' + this.time + ', ' + (this.successful ? '1' : '0') + ', ' + this.failureMessage; + return this.name + ', ' + this.time + ', ' + (this.successful ? '1' : '0') + ', ' + this.failureMessage; } } diff --git a/packages/api/lib/xunit-parser.js b/packages/api/lib/xunit-parser.js new file mode 100644 index 00000000..f2fb8d19 --- /dev/null +++ b/packages/api/lib/xunit-parser.js @@ -0,0 +1,114 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const xmljs = require('xml-js'); +const TestCaseRun = require('../lib/testrun'); + +class Parser { + /** + Parse the xunit test results, converting them to an array of testrun objects. + Each of these testrun objects contains whether the test passed, its name, and + its failure message (if failed). + @param xmlString - The complete xunit string of test results sent to the endpoint + */ + parse (xmlString) { + const obj = xmljs.xml2js(xmlString, { compact: true }); + const tests = []; + // Python doesn't always have a top-level testsuites element. + let testsuites = obj.testsuite; + if (testsuites === undefined) { + testsuites = obj.testsuites.testsuite; + } + if (testsuites === undefined) { + return tests; + } + // If there is only one test suite, put it into an array to make it iterable. + if (!Array.isArray(testsuites)) { + testsuites = [testsuites]; + } + for (const suite of testsuites) { + // Ruby doesn't always have _attributes. + let testsuiteName = suite._attributes ? suite._attributes.name : undefined; + + // Get rid of github.com/orgName/repoName/ + testsuiteName = this.trim(testsuiteName); + + let testcases = suite.testcase; + // If there were no tests in the package, continue. + if (testcases === undefined) { + continue; + } + // If there is only one test case, put it into an array to make it iterable. + if (!Array.isArray(testcases)) { + testcases = [testcases]; + } + + for (const testcase of testcases) { + // Ignore skipped tests. They didn't pass and they didn't fail. + if (testcase.skipped !== undefined) { + continue; + } + + const failure = testcase.failure; + const error = testcase.error; + + const okayMessage = (failure === undefined && error === undefined) ? 'ok' : 'not ok'; + let name = testcase._attributes.name; + + if (testsuiteName.length > 0) { + name = testsuiteName + '/' + name; + } + + const testCaseRun = new TestCaseRun(okayMessage, name); + + if (!testCaseRun.successful) { + // Here we must have a failure or an error. + let log = (failure === undefined) ? error._text : failure._text; + // Java puts its test logs in a CDATA element. + if (log === undefined) { + log = failure._cdata; + } + + testCaseRun.failureMessage = log; + } + + tests.push(testCaseRun); + } + } + return tests; + } + + /** + The url contains 'github.com/:org/:repo', optionally followed by a path. + This method trims the url down to only the string following `repo`, which may be empty. + @param url - the Github url of where a test came from + */ + trim (url) { + const updated = url.replace(/(.)*github.com\/[a-zA-Z-]*\/[a-zA-Z-]*/g, ''); + if (updated.length > 1) return updated.substring(1); // Get rid of starting `/` + return updated; + } + + // IMPORTANT: All values that will be used as keys in Firestore must be escaped with the firestoreEncode function + /** + Parse the build data and convert it into a JSON format that can easily be read and stored. + @param metadata - contains all data sent to the endpoint besides the actual test results. + */ + cleanXunitBuildInfo (metadata) { + return {}; + } +} + +const parserHandler = new Parser(); +module.exports = parserHandler; diff --git a/packages/api/package.json b/packages/api/package.json index e2d542af..a07a4eb7 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -33,7 +33,8 @@ "express-session": "^1.17.1", "moment": "^2.27.0", "tap-parser": "^10.0.1", - "uuid": "^8.2.0" + "uuid": "^8.2.0", + "xml-js": "^1.6.11" }, "devDependencies": { "c8": "^7.1.2", diff --git a/packages/api/src/add-build.js b/packages/api/src/add-build.js index 6b45684f..20ef4d39 100644 --- a/packages/api/src/add-build.js +++ b/packages/api/src/add-build.js @@ -278,4 +278,4 @@ async function updateRepoDoc (buildInfo, computedData, mostRecent, dbRepo) { // db read operation to fix this. This would only be visible when ordering by priority on org page. } -module.exports = addBuild; +module.exports = { addBuild }; diff --git a/packages/api/src/post-build.js b/packages/api/src/post-build.js index de03a945..aec96a62 100644 --- a/packages/api/src/post-build.js +++ b/packages/api/src/post-build.js @@ -16,9 +16,10 @@ // NOTE: relies on global.headCollection to be the high level repository -const addBuild = require('../src/add-build'); +const AddBuildHandler = require('../src/add-build'); const TestCaseRun = require('../lib/testrun'); -var Parser = require('tap-parser'); +const TapParser = require('tap-parser'); +const xunitParser = require('../lib/xunit-parser'); const Readable = require('stream').Readable; const firebaseEncode = require('../lib/firebase-encode'); const { InvalidParameterError, UnauthorizedError, handleError } = require('../lib/errors'); @@ -96,7 +97,7 @@ class PostBuildHandler { if (typeof x.ok !== 'boolean' || !x.id || !x.name) { throw new InvalidParameterError('Missing All Test Case Info'); } - const testcase = new TestCaseRun(x.ok ? 'ok' : 'not ok', x.id, x.name); + const testcase = new TestCaseRun(x.ok ? 'ok' : 'not ok', x.name); // wrap failure message generation in try so still works if ids arent sequential try { @@ -121,7 +122,7 @@ class PostBuildHandler { switch (fileType) { case 'TAP': { var data = []; - var p = new Parser(); + var p = new TapParser(); p.on('result', function (assert) { data.push(assert); @@ -169,7 +170,7 @@ class PostBuildHandler { } // IMPORTANT: All values that will be used as keys in Firestore must be escaped with the firestoreEncode function - static cleanBuildInfo (metadata) { + static cleanTapBuildInfo (metadata) { const timestampNumb = Date.parse(metadata.timestamp); const timestamp = isNaN(timestampNumb) ? new Date() : new Date(timestampNumb); @@ -232,7 +233,7 @@ class PostBuildHandler { throw new UnauthorizedError('Flaky does not store tests for private repos'); } - await addBuild(PostBuildHandler.removeDuplicateTestCases(testCases), buildInfo, this.client, global.headCollection); + await AddBuildHandler.addBuild(PostBuildHandler.removeDuplicateTestCases(testCases), buildInfo, this.client, global.headCollection); res.send({ message: 'successfully added build' }); } catch (err) { handleError(res, err); @@ -247,7 +248,7 @@ class PostBuildHandler { throw new UnauthorizedError('Flaky does not store tests for private repos'); } - const buildInfo = PostBuildHandler.cleanBuildInfo(req.body.metadata); // Different line. The metadata object is the same as addbuild, already validated + const buildInfo = PostBuildHandler.cleanTapBuildInfo(req.body.metadata); // Different line. The metadata object is the same as addbuild, already validated req.body.data = await PostBuildHandler.flattenTap(req.body.data); const parsedRaw = await PostBuildHandler.parseRawOutput(req.body.data, req.body.type); @@ -258,7 +259,25 @@ class PostBuildHandler { throw new UnauthorizedError('Must have valid Github Token to post build'); } - await addBuild(PostBuildHandler.removeDuplicateTestCases(testCases), buildInfo, this.client, global.headCollection); + await AddBuildHandler.addBuild(PostBuildHandler.removeDuplicateTestCases(testCases), buildInfo, this.client, global.headCollection); + res.send({ message: 'successfully added build' }); + } catch (err) { + handleError(res, err); + } + }); + + // endpoint expects the the required buildinfo to be in req.body.metadata to already exist and be properly formatted. + // required keys in the req.body.metadata are the inputs for addBuild in src/add-build.js + this.app.post('/api/build/xml', async (req, res, next) => { + try { + if (req.headers.authorization !== process.env.PRIVATE_POSTING_TOKEN) { + throw new UnauthorizedError('Invalid Secret. Only Google Employees may use this endpoint.'); + } + + const testCases = xunitParser.parse(req.body.data); + const buildInfo = xunitParser.cleanXunitBuildInfo(req.body.metadata); + + await AddBuildHandler.addBuild(PostBuildHandler.removeDuplicateTestCases(testCases), buildInfo, this.client, global.headCollection); res.send({ message: 'successfully added build' }); } catch (err) { handleError(res, err); diff --git a/packages/api/test/addbuild-getbuild.js b/packages/api/test/addbuild-getbuild.js index fe8ea0e1..adb239b5 100644 --- a/packages/api/test/addbuild-getbuild.js +++ b/packages/api/test/addbuild-getbuild.js @@ -19,7 +19,7 @@ const client = require('../src/firestore.js'); const firebaseEncode = require('../lib/firebase-encode'); const TestCaseRun = require('../lib/testrun'); -const addBuild = require('../src/add-build'); +const { addBuild } = require('../src/add-build'); const { deleteTest, deleteRepo } = require('../lib/deleter'); const fetch = require('node-fetch'); @@ -47,10 +47,10 @@ const buildInfo = [ }, timestamp: new Date('01/01/2000'), testCases: [ - new TestCaseRun('ok', 1, 'a/1'), - new TestCaseRun('not ok', 2, 'a/2'), - new TestCaseRun('ok', 3, 'a/3'), - new TestCaseRun('not ok', 4, 'a/4') + new TestCaseRun('ok', 'a/1'), + new TestCaseRun('not ok', 'a/2'), + new TestCaseRun('ok', 'a/3'), + new TestCaseRun('not ok', 'a/4') ], description: 'nodejs repository', buildmessage: 'Workflow - 1' @@ -70,8 +70,8 @@ const buildInfo = [ }, timestamp: new Date('01/01/2001'), testCases: [ - new TestCaseRun('ok', 1, 'a/1'), - new TestCaseRun('ok', 2, 'a/2') // this test is now passing + new TestCaseRun('ok', 'a/1'), + new TestCaseRun('ok', 'a/2') // this test is now passing ], description: 'nodejs repository', buildmessage: 'Workflow - 2' @@ -91,8 +91,8 @@ const buildInfo = [ }, timestamp: new Date('01/01/2002'), testCases: [ - new TestCaseRun('not ok', 1, 'a/5'), - new TestCaseRun('not ok', 2, 'a/2') // this test is now failing + new TestCaseRun('not ok', 'a/5'), + new TestCaseRun('not ok', 'a/2') // this test is now failing ], description: 'None', buildmessage: 'Workflow - 1' @@ -102,7 +102,7 @@ const buildInfo = [ buildInfo[2].testCases[0].failureMessage = 'Error message'; buildInfo[2].testCases[1].failureMessage = 'Error message'; -describe.only('Add-Build', () => { +describe('Add-Build', () => { before(async () => { global.headCollection = 'testing/' + TESTING_COLLECTION_BASE + uuidv4() + '/repos'; // random collection name for concurrent testing }); @@ -434,8 +434,8 @@ describe.only('Add-Build', () => { }, timestamp: new Date('01/01/2004'), testCases: [ - new TestCaseRun('not ok', 1, 'a/5'), - new TestCaseRun('not ok', 2, 'a/2') // this test is now passing + new TestCaseRun('not ok', 'a/5'), + new TestCaseRun('not ok', 'a/2') // this test is now passing ], description: 'nodejs repository', buildmessage: 'Workflow - 42' diff --git a/packages/api/test/analytics-test.js b/packages/api/test/analytics-test.js index 29c07a84..eb69ff04 100644 --- a/packages/api/test/analytics-test.js +++ b/packages/api/test/analytics-test.js @@ -18,7 +18,7 @@ const { v4: uuidv4 } = require('uuid'); const client = require('../src/firestore.js'); const TestCaseRun = require('../lib/testrun'); -const addBuild = require('../src/add-build'); +const { addBuild } = require('../src/add-build'); const { deleteRepo } = require('../lib/deleter'); const buildInfo = { diff --git a/packages/api/test/fixtures/empty_results.xml b/packages/api/test/fixtures/empty_results.xml new file mode 100644 index 00000000..4fe0daf9 --- /dev/null +++ b/packages/api/test/fixtures/empty_results.xml @@ -0,0 +1,2 @@ + + diff --git a/packages/api/test/fixtures/go_failure_group.xml b/packages/api/test/fixtures/go_failure_group.xml new file mode 100644 index 00000000..05d23e9c --- /dev/null +++ b/packages/api/test/fixtures/go_failure_group.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + panic: runtime error: invalid memory address or nil pointer dereference /usr/local/go/src/testing/testing.go:874 +0x3a3 /usr/local/go/src/runtime/panic.go:679 +0x1b2 /go/pkg/mod/cloud.google.com/go/bigquery@v1.3.0/iterator.go:106 +0x37 /tmpfs/src/github/golang-samples/bigquery/snippets/querying/bigquery_query_legacy_large_results.go:59 +0x419 /tmpfs/src/github/golang-samples/bigquery/snippets/querying/integration_test.go:90 +0xa5 /usr/local/go/src/testing/testing.go:909 +0xc9 /usr/local/go/src/testing/testing.go:960 +0x350 + + + + + + + + + + + + diff --git a/packages/api/test/fixtures/go_skip.xml b/packages/api/test/fixtures/go_skip.xml new file mode 100644 index 00000000..52ad9b7c --- /dev/null +++ b/packages/api/test/fixtures/go_skip.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/packages/api/test/fixtures/java_one_failed.xml b/packages/api/test/fixtures/java_one_failed.xml new file mode 100644 index 00000000..c71cfa18 --- /dev/null +++ b/packages/api/test/fixtures/java_one_failed.xml @@ -0,0 +1,113 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + but was: + at org.junit.Assert.fail(Assert.java:89) + at org.junit.Assert.failNotEquals(Assert.java:835) + at org.junit.Assert.assertEquals(Assert.java:120) + at org.junit.Assert.assertEquals(Assert.java:146) + at com.google.cloud.vision.it.ITSystemTest.detectSafeSearchGcsTest(ITSystemTest.java:404) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:498) + at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) + at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) + at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) + at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) + at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) + at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) + at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) + at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) + at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) + at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) + at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) + at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) + at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) + at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) + at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) + at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) + at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) + at org.junit.runners.ParentRunner.run(ParentRunner.java:413) + at org.junit.runners.Suite.runChild(Suite.java:128) + at org.junit.runners.Suite.runChild(Suite.java:27) + at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) + at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) + at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) + at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) + at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) + at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) + at org.junit.runners.ParentRunner.run(ParentRunner.java:413) + at org.apache.maven.surefire.junitcore.JUnitCore.run(JUnitCore.java:55) + at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.createRequestAndRun(JUnitCoreWrapper.java:137) + at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.executeEager(JUnitCoreWrapper.java:107) + at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.execute(JUnitCoreWrapper.java:83) + at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.execute(JUnitCoreWrapper.java:75) + at org.apache.maven.surefire.junitcore.JUnitCoreProvider.invoke(JUnitCoreProvider.java:158) + at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:377) + at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:138) + at org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:465) + at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:451) +]]> + + + \ No newline at end of file diff --git a/packages/api/test/fixtures/java_one_passed.xml b/packages/api/test/fixtures/java_one_passed.xml new file mode 100644 index 00000000..37d07532 --- /dev/null +++ b/packages/api/test/fixtures/java_one_passed.xml @@ -0,0 +1,63 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/packages/api/test/fixtures/many_failed_same_pkg.xml b/packages/api/test/fixtures/many_failed_same_pkg.xml new file mode 100644 index 00000000..d6993408 --- /dev/null +++ b/packages/api/test/fixtures/many_failed_same_pkg.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + main_test.go:234: failed to create bucket ("golang-samples-tests-8-storage-buckets-tests"): Post https://storage.googleapis.com/storage/v1/b?alt=json&prettyPrint=false&project=golang-samples-tests-8: read tcp 10.142.0.112:33618->108.177.12.128:443: read: connection reset by peer + + + main_test.go:242: failed to enable uniform bucket-level access ("golang-samples-tests-8-storage-buckets-tests"): googleapi: Error 404: Not Found, notFound + + + + + + + + + diff --git a/packages/api/test/fixtures/no_tests.xml b/packages/api/test/fixtures/no_tests.xml new file mode 100644 index 00000000..162edf53 --- /dev/null +++ b/packages/api/test/fixtures/no_tests.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/packages/api/test/fixtures/node_group.xml b/packages/api/test/fixtures/node_group.xml new file mode 100644 index 00000000..52c7def6 --- /dev/null +++ b/packages/api/test/fixtures/node_group.xml @@ -0,0 +1,162 @@ + + + + + expected 'Deleted individual rows in Albums.\n5 records deleted from Singers.\n2 records deleted from Singers.\n0 records deleted from Singers.\n' to include '3 records deleted from Singers.' + AssertionError: expected 'Deleted individual rows in Albums.\n5 records deleted from Singers.\n2 records deleted from Singers.\n0 records deleted from Singers.\n' to include '3 records deleted from Singers.' + at Context.it (system-test/spanner.test.js:198:12) + expected '' to match /SingerId: 1, AlbumId: 1, AlbumTitle: Total Junk/ + AssertionError: expected '' to match /SingerId: 1, AlbumId: 1, AlbumTitle: Total Junk/ + at Context.it (system-test/spanner.test.js:210:12) + expected '' to match /SingerId: 1, AlbumId: 1, AlbumTitle: Total Junk/ + AssertionError: expected '' to match /SingerId: 1, AlbumId: 1, AlbumTitle: Total Junk/ + at Context.it (system-test/spanner.test.js:218:12) + + expected '' to match /Updated data\./ + AssertionError: expected '' to match /Updated data\./ + at Context.it (system-test/spanner.test.js:235:12) + expected '' to match /SingerId: 1, AlbumId: 1, AlbumTitle: Total Junk, MarketingBudget: 100000/ + AssertionError: expected '' to match /SingerId: 1, AlbumId: 1, AlbumTitle: Total Junk, MarketingBudget: 100000/ + at Context.it (system-test/spanner.test.js:246:12) + expected '' to match /SingerId: 1, AlbumId: 1, MarketingBudget: 100000/ + AssertionError: expected '' to match /SingerId: 1, AlbumId: 1, MarketingBudget: 100000/ + at Context.it (system-test/spanner.test.js:261:12) + + + expected '' to match /AlbumId: 2, AlbumTitle: Go, Go, Go, MarketingBudget:/ + AssertionError: expected '' to match /AlbumId: 2, AlbumTitle: Go, Go, Go, MarketingBudget:/ + at Context.it (system-test/spanner.test.js:288:12) + expected '' to match /AlbumId: 1, AlbumTitle: Total Junk, MarketingBudget:/ + AssertionError: expected '' to match /AlbumId: 1, AlbumTitle: Total Junk, MarketingBudget:/ + at Context.it (system-test/spanner.test.js:302:12) + expected '' to match /AlbumId: 1, AlbumTitle: Total Junk/ + AssertionError: expected '' to match /AlbumId: 1, AlbumTitle: Total Junk/ + at Context.it (system-test/spanner.test.js:317:12) + expected '' to match /AlbumId: 1, AlbumTitle: Total Junk/ + AssertionError: expected '' to match /AlbumId: 1, AlbumTitle: Total Junk/ + at Context.it (system-test/spanner.test.js:325:12) + expected '' to match /AlbumId: 2, AlbumTitle: Forever Hold your Peace, MarketingBudget:/ + AssertionError: expected '' to match /AlbumId: 2, AlbumTitle: Forever Hold your Peace, MarketingBudget:/ + at Context.it (system-test/spanner.test.js:333:12) + expected '' to match /AlbumId: 2, AlbumTitle: Forever Hold your Peace, MarketingBudget:/ + AssertionError: expected '' to match /AlbumId: 2, AlbumTitle: Forever Hold your Peace, MarketingBudget:/ + at Context.it (system-test/spanner.test.js:344:12) + expected 'Successfully executed read-only transaction.\n' to match /SingerId: 1, AlbumId: 1, AlbumTitle: Total Junk/ + AssertionError: expected 'Successfully executed read-only transaction.\n' to match /SingerId: 1, AlbumId: 1, AlbumTitle: Total Junk/ + at Context.it (system-test/spanner.test.js:355:12) + expected '' to match /The first album's marketing budget: 100000/ + AssertionError: expected '' to match /The first album's marketing budget: 100000/ + at Context.it (system-test/spanner.test.js:364:12) + + + + expected '' to match /Updated data\./ + AssertionError: expected '' to match /Updated data\./ + at Context.it (system-test/spanner.test.js:426:12) + expected '' to match /SingerId: 1, AlbumId: 1, MarketingBudget: 1000000, LastUpdateTime:/ + AssertionError: expected '' to match /SingerId: 1, AlbumId: 1, MarketingBudget: 1000000, LastUpdateTime:/ + at Context.it (system-test/spanner.test.js:434:12) + + expected '' to match /Inserted data\./ + AssertionError: expected '' to match /Inserted data\./ + at Context.it (system-test/spanner.test.js:465:12) + expected '' to match /SingerId: 1, VenueId: 4, EventDate:/ + AssertionError: expected '' to match /SingerId: 1, VenueId: 4, EventDate:/ + at Context.it (system-test/spanner.test.js:473:12) + + + + + + + expected 'Successfully updated 0 record.\n' to match /Successfully updated 1 record/ + AssertionError: expected 'Successfully updated 0 record.\n' to match /Successfully updated 1 record/ + at Context.it (system-test/spanner.test.js:536:12) + expected 'Successfully deleted 0 record.\n' to match /Successfully deleted 1 record\./ + AssertionError: expected 'Successfully deleted 0 record.\n' to match /Successfully deleted 1 record\./ + at Context.it (system-test/spanner.test.js:544:12) + expected 'Successfully updated 0 records.\n' to match /Successfully updated 2 records/ + AssertionError: expected 'Successfully updated 0 records.\n' to match /Successfully updated 2 records/ + at Context.it (system-test/spanner.test.js:552:12) + + + + + expected '' to match /Successfully executed read-write transaction using DML to transfer 200000 from Album 2 to Album 1/ + AssertionError: expected '' to match /Successfully executed read-write transaction using DML to transfer 200000 from Album 2 to Album 1/ + at Context.it (system-test/spanner.test.js:592:12) + expected 'Successfully updated 0 records.\n' to match /Successfully updated 3 records/ + AssertionError: expected 'Successfully updated 0 records.\n' to match /Successfully updated 3 records/ + at Context.it (system-test/spanner.test.js:603:12) + + Command failed: node dml.js updateUsingBatchDml test-instance-1591914381325 test-database-1591914381325 long-door-651 + ERROR: { Error: Parent row for row [1,3] in table Albums is missing. Row cannot be written. + at Immediate.request (/tmpfs/src/github/nodejs-spanner/build/src/transaction.js:1082:31) + at runCallback (timers.js:706:11) + at tryOnImmediate (timers.js:676:5) + at processImmediate (timers.js:658:5) + code: 5, + metadata: Metadata { internalRepr: Map {}, options: {} }, + rowCounts: [] } + dml.js updateUsingBatchDml <instanceName> <databaseName> <projectId> + + Insert and Update records using Batch DML. + + Options: + --version Show version number [boolean] + --help Show help [boolean] + + { Error: Parent row for row [1,3] in table Albums is missing. Row cannot be written. + at Immediate.request (/tmpfs/src/github/nodejs-spanner/build/src/transaction.js:1082:31) + at runCallback (timers.js:706:11) + at tryOnImmediate (timers.js:676:5) + at processImmediate (timers.js:658:5) + code: 5, + metadata: Metadata { internalRepr: Map {}, options: {} }, + rowCounts: [] } + + Error: Command failed: node dml.js updateUsingBatchDml test-instance-1591914381325 test-database-1591914381325 long-door-651 + ERROR: { Error: Parent row for row [1,3] in table Albums is missing. Row cannot be written. + at Immediate.request (/tmpfs/src/github/nodejs-spanner/build/src/transaction.js:1082:31) + code: 5, + metadata: Metadata { internalRepr: Map {}, options: {} }, + rowCounts: [] } + dml.js updateUsingBatchDml <instanceName> <databaseName> <projectId> + + Insert and Update records using Batch DML. + + Options: + --version Show version number [boolean] + --help Show help [boolean] + + { Error: Parent row for row [1,3] in table Albums is missing. Row cannot be written. + at Immediate.request (/tmpfs/src/github/nodejs-spanner/build/src/transaction.js:1082:31) + code: 5, + metadata: Metadata { internalRepr: Map {}, options: {} }, + rowCounts: [] } + + at checkExecSyncError (child_process.js:629:11) + at Object.execSync (child_process.js:666:13) + at execSync (system-test/spanner.test.js:23:28) + at Context.it (system-test/spanner.test.js:616:20) + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/api/test/fixtures/node_group_pass.xml b/packages/api/test/fixtures/node_group_pass.xml new file mode 100644 index 00000000..bab1c565 --- /dev/null +++ b/packages/api/test/fixtures/node_group_pass.xml @@ -0,0 +1,4 @@ + + + + diff --git a/packages/api/test/fixtures/node_one_failed.xml b/packages/api/test/fixtures/node_one_failed.xml new file mode 100644 index 00000000..628a4b10 --- /dev/null +++ b/packages/api/test/fixtures/node_one_failed.xml @@ -0,0 +1,7 @@ + + + + expected 'Deleted individual rows in Albums.\n5 records deleted from Singers.\n2 records deleted from Singers.\n0 records deleted from Singers.\n' to include '3 records deleted from Singers.' + AssertionError: expected 'Deleted individual rows in Albums.\n5 records deleted from Singers.\n2 records deleted from Singers.\n0 records deleted from Singers.\n' to include '3 records deleted from Singers.' + at Context.it (system-test/spanner.test.js:198:12) + diff --git a/packages/api/test/fixtures/one_failed.xml b/packages/api/test/fixtures/one_failed.xml new file mode 100644 index 00000000..3e90b6ed --- /dev/null +++ b/packages/api/test/fixtures/one_failed.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + +snippet_test.go:242: got output ""; want it to contain "4 Venue 4" snippet_test.go:243: got output ""; want it to contain "19 Venue 19" snippet_test.go:244: got output ""; want it to contain "42 Venue 42" + + + + \ No newline at end of file diff --git a/packages/api/test/fixtures/passed.xml b/packages/api/test/fixtures/passed.xml new file mode 100644 index 00000000..f969bc3e --- /dev/null +++ b/packages/api/test/fixtures/passed.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/packages/api/test/fixtures/python_one_error.xml b/packages/api/test/fixtures/python_one_error.xml new file mode 100644 index 00000000..f06e78a7 --- /dev/null +++ b/packages/api/test/fixtures/python_one_error.xml @@ -0,0 +1,6 @@ +Traceback (most recent call last): + File "/workspace/memorystore/redis/cloud_run_deployment/e2e_test.py", line 70, in services + subprocess.run( + File "/usr/local/lib/python3.8/subprocess.py", line 512, in run + raise CalledProcessError(retcode, process.args, +subprocess.CalledProcessError: Command '['gcloud', 'redis', 'instances', 'create', 'test-instance-44d74c74c5', '--region=us-central1', '--network', 'test-network-44d74c74c5', '--project', 'python-docs-samples-tests']' returned non-zero exit status 1. \ No newline at end of file diff --git a/packages/api/test/fixtures/python_one_failed.xml b/packages/api/test/fixtures/python_one_failed.xml new file mode 100644 index 00000000..e967ed68 --- /dev/null +++ b/packages/api/test/fixtures/python_one_failed.xml @@ -0,0 +1,12 @@ + + + + + +Traceback (most recent call last): + File "/tmpfs/src/github/python-docs-samples/appengine/flexible/datastore/main_test.py", line 22, in test_index + ... + + + + diff --git a/packages/api/test/fixtures/python_one_passed.xml b/packages/api/test/fixtures/python_one_passed.xml new file mode 100644 index 00000000..a5afbc82 --- /dev/null +++ b/packages/api/test/fixtures/python_one_passed.xml @@ -0,0 +1,4 @@ + + + + diff --git a/packages/api/test/fixtures/ruby_one_failed.xml b/packages/api/test/fixtures/ruby_one_failed.xml new file mode 100644 index 00000000..2300e410 --- /dev/null +++ b/packages/api/test/fixtures/ruby_one_failed.xml @@ -0,0 +1,23 @@ + + + + + + + + + + +/usr/local/bundle/gems/minitest-5.14.1/lib/minitest/assertions.rb:183:in `assert' +/tmpfs/src/github/ruby-docs-samples/logging/acceptance/sample_test.rb:126:in `block (3 levels) in <top (required)>' +/usr/local/bundle/gems/minitest-5.14.1/lib/minitest/test.rb:98:in `block (3 levels) in run' +/usr/local/bundle/gems/minitest-5.14.1/lib/minitest/test.rb:195:in `capture_exceptions' +/usr/local/bundle/gems/minitest-5.14.1/lib/minitest/test.rb:95:in `block (2 levels) in run' +/usr/local/bundle/gems/minitest-5.14.1/lib/minitest.rb:272:in `time_it' +/usr/local/bundle/gems/minitest-5.14.1/lib/minitest/test.rb:94:in `block in run' +/usr/local/bundle/gems/minitest-5.14.1/lib/minitest.rb:367:in `on_signal' +/usr/local/bundle/gems/minitest-5.14.1/lib/minitest/test.rb:211:in `with_info_handler' +/usr/local/bundle/gems/minitest-5.14.1/lib/minitest/test.rb:93:in `run' +/usr/local/bundle/gems/minitest-5.14.1/lib/minitest.rb:1029:in `run_one_method' +/usr/local/bundle/gems/minitest-5.14.1/lib/minitest/parallel.rb:33:in `block (2 levels) in start' + \ No newline at end of file diff --git a/packages/api/test/get-batches.js b/packages/api/test/get-batches.js index 18f877a1..e4043a89 100644 --- a/packages/api/test/get-batches.js +++ b/packages/api/test/get-batches.js @@ -20,7 +20,7 @@ const sinon = require('sinon'); const assert = require('assert'); const fetch = require('node-fetch'); const firebaseEncode = require('../lib/firebase-encode'); -const addBuild = require('../src/add-build'); +const { addBuild } = require('../src/add-build'); const { deleteRepo } = require('../lib/deleter'); const client = require('../src/firestore'); diff --git a/packages/api/test/post-build-test.js b/packages/api/test/post-build-test.js index f2bda51a..179a41aa 100644 --- a/packages/api/test/post-build-test.js +++ b/packages/api/test/post-build-test.js @@ -23,20 +23,24 @@ const EXAMPLE_TAP_MANGLED = fs.readFileSync(path.join(__dirname, 'res/mangledtap const EXAMPLE_TAP_NESTED = fs.readFileSync(path.join(__dirname, 'res/nestedtap.tap'), 'utf8'); const EXAMPLE_STUFF_ON_TOP = fs.readFileSync(path.join(__dirname, 'res/stuffontoptap.tap'), 'utf8'); -const { describe, before, after, it } = require('mocha'); +const { describe, before, after, it, afterEach } = require('mocha'); const { v4: uuidv4 } = require('uuid'); const firebaseEncode = require('../lib/firebase-encode'); const nock = require('nock'); +const sinon = require('sinon'); const validNockResponse = require('./res/sample-validate-resp.json'); const { deleteRepo } = require('../lib/deleter'); const PostBuildHandler = require('../src/post-build.js'); +const AddBuildHandler = require('../src/add-build'); const client = require('../src/firestore.js'); const assert = require('assert'); const fetch = require('node-fetch'); +const xunitParser = require('../lib/xunit-parser'); + nock.disableNetConnect(); nock.enableNetConnect(/^(?!.*github\.com).*$/); // only disable requests on github.com @@ -73,12 +77,10 @@ describe('Posting Builds', () => { // two failed assert(!testcases[2].successful); - assert.strictEqual(testcases[2].number, 3); assert(testcases[2].failureMessage.includes('AssertionError')); // five failed assert(!testcases[5].successful); - assert.strictEqual(testcases[5].number, 6); assert(testcases[5].failureMessage.includes('AssertionError')); }); @@ -261,6 +263,56 @@ describe('Posting Builds', () => { assert.strictEqual(result.data().environment.ref, 'master'); }); + describe('xunit endpoint', async () => { + const sandbox = sinon.createSandbox(); + + afterEach(() => { + sandbox.restore(); + }); + + it('does not allow a post if no password is included', async () => { + sandbox.stub(AddBuildHandler, 'addBuild'); + + const bodyData = fs.readFileSync(require.resolve('./fixtures/passed.xml'), 'utf8'); + process.env.PRIVATE_POSTING_TOKEN = 'hello'; + + const resp = await fetch('http://127.0.0.1:3000/api/build/xml', { + method: 'post', + body: JSON.stringify({ + data: bodyData, + metadata: {} + }), + headers: { 'Content-Type': 'application/json' } + }); + + assert.strictEqual(resp.status, 401); + assert.strictEqual(resp.statusText, 'Unauthorized'); + }); + + it('calls addBuild with appropriate data when authentication token is included', async () => { + const addBuildStub = sinon.stub(AddBuildHandler, 'addBuild'); + sandbox.stub(xunitParser, 'parse').returns([]); + sandbox.stub(xunitParser, 'cleanXunitBuildInfo').returns({}); + + const bodyData = fs.readFileSync(require.resolve('./fixtures/passed.xml'), 'utf8'); + process.env.PRIVATE_POSTING_TOKEN = 'hello'; + + const resp = await fetch('http://127.0.0.1:3000/api/build/xml', { + method: 'post', + body: JSON.stringify({ + data: bodyData, + metadata: {} + }), + headers: { 'content-type': 'application/json', Authorization: process.env.PRIVATE_POSTING_TOKEN } + }); + + assert.strictEqual(resp.status, 200); + assert(addBuildStub.calledWith([], {})); + + addBuildStub.restore(); + }); + }); + after(async () => { var parsedPayload = JSON.parse(EXAMPLE_PAYLOAD); var parsedPayloadRaw = JSON.parse(EXAMPLE_PAYLOAD_RAW); diff --git a/packages/api/test/server-test.js b/packages/api/test/server-test.js index e2156304..6731b8b7 100644 --- a/packages/api/test/server-test.js +++ b/packages/api/test/server-test.js @@ -24,15 +24,11 @@ const auth = require('../src/auth.js'); const repo = require('../src/repository.js'); describe('flaky express server', () => { - let stubs = []; + const sandbox = sinon.createSandbox(); const frontendUrl = 'https://flaky-dashboard.web.app/home'; afterEach(() => { - /** Cleanup **/ - stubs.forEach(stubbed => { - stubbed.restore(); - }); - stubs = []; + sandbox.restore(); }); it('should have the mocked environment variables', () => { @@ -41,7 +37,7 @@ describe('flaky express server', () => { describe('get /repo to delete a test', async () => { it('generates a GitHub redirect', async () => { - stubs.push(sinon.stub(repo, 'storeTicket').returns(true)); + sandbox.stub(repo, 'storeTicket').returns(true); const resp = await fetch('http://0.0.0.0:3000/api/repo/my-org/my-repo/test/deleteurl?testname=my-test&redirect=' + process.env.FRONTEND_URL, { method: 'GET' }); @@ -51,7 +47,6 @@ describe('flaky express server', () => { it('stores correct information in the ticket', async () => { const stubbed = sinon.stub(repo, 'storeTicket'); - stubs.push(stubbed); await fetch('http://0.0.0.0:3000/api/repo/my-org/my-repo/test/deleteurl?testname=my-test&redirect=' + process.env.FRONTEND_URL, { method: 'GET' @@ -64,12 +59,14 @@ describe('flaky express server', () => { testName: 'my-test', redirect: process.env.FRONTEND_URL })); + + stubbed.restore(); }); }); describe('get /repo to delete a repository', async () => { it('generates a GitHub redirect', async () => { - stubs.push(sinon.stub(repo, 'storeTicket').returns(true)); + sandbox.stub(repo, 'storeTicket').returns(true); const resp = await fetch('http://0.0.0.0:3000/api/repo/my-org/my-repo/deleteurl?redirect=' + process.env.FRONTEND_URL, { method: 'GET' }); @@ -79,7 +76,6 @@ describe('flaky express server', () => { it('stores correct information in the ticket', async () => { const stubbed = sinon.stub(repo, 'storeTicket'); - stubs.push(stubbed); await fetch('http://0.0.0.0:3000/api/repo/my-org/my-repo/deleteurl?redirect=' + process.env.FRONTEND_URL, { method: 'GET' @@ -91,6 +87,8 @@ describe('flaky express server', () => { repoId: 'my-repo', redirect: process.env.FRONTEND_URL })); + + stubbed.restore(); }); }); @@ -99,17 +97,17 @@ describe('flaky express server', () => { /** Stubbing **/ const queryObject = querystring.stringify({ access_token: 'fake-access-token' }); - stubs.push(sinon.stub(auth, 'retrieveAccessToken').returns(queryObject)); + sandbox.stub(auth, 'retrieveAccessToken').returns(queryObject); - stubs.push(sinon.stub(auth, 'retrieveUserPermission').returns('write')); + sandbox.stub(auth, 'retrieveUserPermission').returns('write'); - stubs.push(sinon.stub(repo, 'performTicketIfAllowed').returns(true)); + sandbox.stub(repo, 'performTicketIfAllowed').returns(true); const fakeState = 'testing-state'; - stubs.push(sinon.stub(repo, 'getTicket').returns({ + sandbox.stub(repo, 'getTicket').returns({ state: fakeState, redirect: process.env.FRONTEND_URL - })); + }); /** Testing **/ const resp = await fetch('http://0.0.0.0:3000/api/callback?state=' + fakeState + '&code=ANYTHING', { diff --git a/packages/api/test/xunit-parser-test.js b/packages/api/test/xunit-parser-test.js new file mode 100644 index 00000000..1b7cc3a2 --- /dev/null +++ b/packages/api/test/xunit-parser-test.js @@ -0,0 +1,51 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const { describe, it } = require('mocha'); +const assert = require('assert'); +const parser = require('../lib/xunit-parser'); +const { readFileSync } = require('fs'); + +describe('xunit-parser-test', () => { + it('stores the correct number of tests', () => { + const tests = parser.parse(readFileSync(require.resolve('./fixtures/one_failed.xml'), 'utf8')); + assert.strictEqual(tests.length, 4); + }); + + it('stores a passing test with the correct values', () => { + const tests = parser.parse(readFileSync(require.resolve('./fixtures/one_failed.xml'), 'utf8')); + const test = tests[0]; + + assert.strictEqual(test.successful, true); + assert.strictEqual(test.name, 'TestBadFiles'); + assert.strictEqual(test.failureMessage, 'Successful'); + }); + + it('stores a failing test with the correct values', () => { + const tests = parser.parse(readFileSync(require.resolve('./fixtures/one_failed.xml'), 'utf8')); + const test = tests[3]; assert.strictEqual(test.successful, false); + assert.strictEqual(test.name, 'spanner/spanner_snippets/TestSample'); + assert.strictEqual(test.failureMessage, '\nsnippet_test.go:242: got output ""; want it to contain "4 Venue 4" snippet_test.go:243: got output ""; want it to contain "19 Venue 19" snippet_test.go:244: got output ""; want it to contain "42 Venue 42"\n'); + }); + + it('handles an empty testsuite', () => { + const tests = parser.parse(readFileSync(require.resolve('./fixtures/empty_results.xml'), 'utf8')); + assert.deepStrictEqual(tests, []); + }); + + it('ignores skipped tests', () => { + const tests = parser.parse(readFileSync(require.resolve('./fixtures/go_skip.xml'), 'utf8')); + assert.strictEqual(tests.length, 1); + }); +});