diff --git a/packages/driver/cypress/integration/commands/angular_spec.js b/packages/driver/cypress/integration/commands/angular_spec.js index 1a7dff5b539d..5118e9c49621 100644 --- a/packages/driver/cypress/integration/commands/angular_spec.js +++ b/packages/driver/cypress/integration/commands/angular_spec.js @@ -28,7 +28,7 @@ describe('src/cy/commands/angular', () => { cy.state('window').angular = this.angular }) - it('throws when cannot find angular', (done) => { + it('throws when cannot find angular', { retries: 2 }, (done) => { delete cy.state('window').angular cy.on('fail', (err) => { diff --git a/packages/driver/src/cy/testConfigOverrides.ts b/packages/driver/src/cy/testConfigOverrides.ts index 28f932f20cdd..290817c6c603 100644 --- a/packages/driver/src/cy/testConfigOverrides.ts +++ b/packages/driver/src/cy/testConfigOverrides.ts @@ -44,20 +44,22 @@ function mutateConfiguration (testConfigOverride, config, env) { return restoreConfigFn } -function getResolvedTestConfigOverride (test) { +// this is called during test onRunnable time +// in order to resolve the test config upfront before test runs +export function getResolvedTestConfigOverride (test) { let curParent = test.parent - const cfgs = [test.cfg] + const testConfig = [test._testConfig] while (curParent) { - if (curParent.cfg) { - cfgs.push(curParent.cfg) + if (curParent._testConfig) { + testConfig.push(curParent._testConfig) } curParent = curParent.parent } - return _.reduceRight(cfgs, (acc, cfg) => _.extend(acc, cfg), {}) + return _.reduceRight(testConfig, (acc, opts) => _.extend(acc, opts), {}) } class TestConfigOverride { @@ -65,7 +67,7 @@ class TestConfigOverride { restoreAndSetTestConfigOverrides (test, config, env) { if (this.restoreTestConfigFn) this.restoreTestConfigFn() - const resolvedTestConfig = getResolvedTestConfigOverride(test) + const resolvedTestConfig = test._testConfig || {} this.restoreTestConfigFn = mutateConfiguration(resolvedTestConfig, config, env) } diff --git a/packages/driver/src/cypress.js b/packages/driver/src/cypress.js index f089af6d30a7..fe7dc45cdd56 100644 --- a/packages/driver/src/cypress.js +++ b/packages/driver/src/cypress.js @@ -134,6 +134,7 @@ class $Cypress { _.extend(this, browserInfo(config)) this.state = $SetterGetter.create({}) + this.originalConfig = _.cloneDeep(config) this.config = $SetterGetter.create(config) this.env = $SetterGetter.create(env) this.getFirefoxGcInterval = $FirefoxForcedGc.createIntervalGetter(this) diff --git a/packages/driver/src/cypress/mocha.js b/packages/driver/src/cypress/mocha.js index d8dd0ee92037..5ad31384b27e 100644 --- a/packages/driver/src/cypress/mocha.js +++ b/packages/driver/src/cypress/mocha.js @@ -33,9 +33,10 @@ const suiteAfterEach = Suite.prototype.afterEach delete window.mocha delete window.Mocha -function invokeFnWithOriginalTitle (ctx, originalTitle, mochaArgs, fn) { +function invokeFnWithOriginalTitle (ctx, originalTitle, mochaArgs, fn, _testConfig) { const ret = fn.apply(ctx, mochaArgs) + ret._testConfig = _testConfig ret.originalTitle = originalTitle return ret @@ -64,13 +65,11 @@ function overloadMochaFnForConfig (fnName, specWindow) { const origFn = subFn ? _fn[subFn] : _fn if (args.length > 2 && _.isObject(args[1])) { - const opts = _.defaults({}, args[1], { - browser: null, - }) + const _testConfig = _.extend({}, args[1]) const mochaArgs = [args[0], args[2]] - const configMatchesBrowser = opts.browser == null || Cypress.isBrowser(opts.browser, `${fnType} config value \`{ browser }\``) + const configMatchesBrowser = _testConfig.browser == null || Cypress.isBrowser(_testConfig.browser, `${fnType} config value \`{ browser }\``) if (!configMatchesBrowser) { // TODO: this would mess up the dashboard since it would be registered as a new test @@ -84,15 +83,15 @@ function overloadMochaFnForConfig (fnName, specWindow) { this.skip() } - return invokeFnWithOriginalTitle(this, originalTitle, mochaArgs, origFn) + return invokeFnWithOriginalTitle(this, originalTitle, mochaArgs, origFn, _testConfig) } - return invokeFnWithOriginalTitle(this, originalTitle, mochaArgs, _fn['skip']) + return invokeFnWithOriginalTitle(this, originalTitle, mochaArgs, _fn['skip'], _testConfig) } const ret = origFn.apply(this, mochaArgs) - ret.cfg = opts + ret._testConfig = _testConfig return ret } @@ -327,7 +326,7 @@ function patchTestClone () { const ret = testClone.apply(this, arguments) // carry over testConfigOverrides - ret.cfg = this.cfg + ret._testConfig = this._testConfig // carry over test.id ret.id = this.id diff --git a/packages/driver/src/cypress/runner.js b/packages/driver/src/cypress/runner.js index 742f98b16d4a..66c2f0732e8e 100644 --- a/packages/driver/src/cypress/runner.js +++ b/packages/driver/src/cypress/runner.js @@ -9,6 +9,7 @@ const $Log = require('./log') const $utils = require('./utils') const $errUtils = require('./error_utils') const $stackUtils = require('./stack_utils') +const { getResolvedTestConfigOverride } = require('../cy/testConfigOverrides') const mochaCtxKeysRe = /^(_runnable|test)$/ const betweenQuotesRe = /\"(.+?)\"/ @@ -18,8 +19,7 @@ const TEST_BEFORE_RUN_EVENT = 'runner:test:before:run' const TEST_AFTER_RUN_EVENT = 'runner:test:after:run' const RUNNABLE_LOGS = 'routes agents commands hooks'.split(' ') -const RUNNABLE_PROPS = 'id order title root hookName hookId err state failedFromHookId body speed type duration wallClockStartedAt wallClockDuration timings file originalTitle invocationDetails final currentRetry retries'.split(' ') - +const RUNNABLE_PROPS = '_testConfig id order title _titlePath root hookName hookId err state failedFromHookId body speed type duration wallClockStartedAt wallClockDuration timings file originalTitle invocationDetails final currentRetry retries'.split(' ') const debug = require('debug')('cypress:driver:runner') const fire = (event, runnable, Cypress) => { @@ -493,6 +493,17 @@ const normalizeAll = (suite, initialTests = {}, setTestsById, setTests, onRunnab setTests(testsArr) } + // generate the diff of the config after spec has been executed + // e.g. config changes via Cypress.config('...') + normalizedSuite.runtimeConfig = {} + _.map(Cypress.config(), (v, key) => { + if (_.isEqual(v, Cypress.originalConfig[key])) { + return null + } + + normalizedSuite.runtimeConfig[key] = v + }) + return normalizedSuite } @@ -548,6 +559,17 @@ const normalize = (runnable, tests, initialTests, onRunnable, onLogsById, getRun // and collections const wrappedRunnable = wrapAll(runnable) + if (runnable.type === 'test') { + const cfg = getResolvedTestConfigOverride(runnable) + + if (_.size(cfg)) { + runnable._testConfig = cfg + wrappedRunnable._testConfig = cfg + } + + wrappedRunnable._titlePath = runnable.titlePath() + } + if (prevAttempts) { wrappedRunnable.prevAttempts = prevAttempts } diff --git a/packages/runner-ct/package.json b/packages/runner-ct/package.json index 69397c6e8eb6..9e3ba1065957 100644 --- a/packages/runner-ct/package.json +++ b/packages/runner-ct/package.json @@ -47,6 +47,7 @@ "@babel/preset-env": "^7.12.1", "@packages/driver": "0.0.0-development", "@percy/cypress": "2.3.4", + "@types/node": "12.12.50", "@types/sockjs-client": "1.1.0", "babel-loader": "8.1.0", "clean-webpack-plugin": "^3.0.0", diff --git a/packages/runner-ct/src/lib/event-manager.js b/packages/runner-ct/src/lib/event-manager.js index 836b254f3f7c..9d2db25b73fe 100644 --- a/packages/runner-ct/src/lib/event-manager.js +++ b/packages/runner-ct/src/lib/event-manager.js @@ -269,6 +269,12 @@ const eventManager = { Cypress.runner.setStartTime(state.startTime) } + if (config.isTextTerminal && !state.currentId) { + // we are in run mode and it's the first load + // store runnables in backend and maybe send to dashboard + return ws.emit('set:runnables:and:maybe:record:tests', runnables, run) + } + if (state.currentId) { // if we have a currentId it means // we need to tell the Cypress to skip @@ -276,11 +282,7 @@ const eventManager = { Cypress.runner.resumeAtTest(state.currentId, state.emissions) } - if (config.isTextTerminal && !state.currentId) { - ws.emit('set:runnables', runnables, run) - } else { - run() - } + run() }) }, }) diff --git a/packages/runner/cypress/support/helpers.js b/packages/runner/cypress/support/helpers.js index 56dedcae05b7..38a77fa3915b 100644 --- a/packages/runner/cypress/support/helpers.js +++ b/packages/runner/cypress/support/helpers.js @@ -260,7 +260,7 @@ function createCypress (defaultOptions = {}) { url: opts.visitUrl, } }) - .withArgs('set:runnables') + .withArgs('set:runnables:and:maybe:record:tests') .callsFake((...args) => { setRunnablesStub(...args) _.last(args)() diff --git a/packages/runner/src/lib/event-manager.js b/packages/runner/src/lib/event-manager.js index 4b4c8495dd01..6cc8104046dc 100644 --- a/packages/runner/src/lib/event-manager.js +++ b/packages/runner/src/lib/event-manager.js @@ -340,6 +340,12 @@ const eventManager = { Cypress.runner.setStartTime(state.startTime) } + if (config.isTextTerminal && !state.currentId) { + // we are in run mode and it's the first load + // store runnables in backend and maybe send to dashboard + return ws.emit('set:runnables:and:maybe:record:tests', runnables, run) + } + if (state.currentId) { // if we have a currentId it means // we need to tell the Cypress to skip @@ -347,11 +353,7 @@ const eventManager = { Cypress.runner.resumeAtTest(state.currentId, state.emissions) } - if (config.isTextTerminal && !state.currentId) { - ws.emit('set:runnables', runnables, run) - } else { - run() - } + run() }) }, }) diff --git a/packages/server/__snapshots__/7_record_spec.js b/packages/server/__snapshots__/7_record_spec.js index ba2569c33648..3d0d12bae3a0 100644 --- a/packages/server/__snapshots__/7_record_spec.js +++ b/packages/server/__snapshots__/7_record_spec.js @@ -268,232 +268,6 @@ Alternatively, you can create a new project using the Desktop Application. https://on.cypress.io/dashboard -` - -exports['e2e record api interaction errors create run 500 warns and does not create or update instances 1'] = ` -Warning: We encountered an error talking to our servers. - -This run will not be recorded. - -This error will not alter the exit code. - -StatusCodeError: 500 - "Internal Server Error" - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass_spec.js) │ - │ Searched: cypress/integration/record_pass* │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: record_pass_spec.js (1 of 1) - - - record pass - ✓ passes - - is pending - - - 1 passing - 1 pending - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 2 │ - │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: true │ - │ Duration: X seconds │ - │ Spec Ran: record_pass_spec.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass_spec.js/yay it passes.png (400x1022) - - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass_spec.js XX:XX 2 1 - 1 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - - - -` - -exports['e2e record api interaction errors create instance does not update instance 1'] = ` - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass_spec.js) │ - │ Searched: cypress/integration/record_pass* │ - │ Params: Tag: false, Group: false, Parallel: false │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - -Warning: We encountered an error talking to our servers. - -This run will not be recorded. - -This error will not alter the exit code. - -StatusCodeError: 500 - "Internal Server Error" - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: record_pass_spec.js (1 of 1) - - - record pass - ✓ passes - - is pending - - - 1 passing - 1 pending - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 2 │ - │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: true │ - │ Duration: X seconds │ - │ Spec Ran: record_pass_spec.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass_spec.js/yay it passes.png (400x1022) - - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass_spec.js XX:XX 2 1 - 1 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - - - -─────────────────────────────────────────────────────────────────────────────────────────────────────── - - Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 - - -` - -exports['e2e record api interaction errors update instance does not update instance stdout 1'] = ` - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass_spec.js) │ - │ Searched: cypress/integration/record_pass* │ - │ Params: Tag: false, Group: false, Parallel: false │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: record_pass_spec.js (1 of 1) - Estimated: 8 seconds - - - record pass - ✓ passes - - is pending - - - 1 passing - 1 pending - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 2 │ - │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: true │ - │ Duration: X seconds │ - │ Estimated: 8 seconds │ - │ Spec Ran: record_pass_spec.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass_spec.js/yay it passes.png (400x1022) - - - (Uploading Results) - -Warning: We encountered an error talking to our servers. - -This run will not be recorded. - -This error will not alter the exit code. - -StatusCodeError: 500 - "Internal Server Error" - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass_spec.js XX:XX 2 1 - 1 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - - - -─────────────────────────────────────────────────────────────────────────────────────────────────────── - - Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 - - ` exports['e2e record api interaction errors update instance stdout warns but proceeds 1'] = ` @@ -774,26 +548,6 @@ exports['e2e record api interaction errors uploading assets warns but proceeds 1 ` -exports['e2e record misconfiguration errors and exits when no browser found 1'] = ` -Can't run because you've entered an invalid browser name. - -Browser: 'browserDoesNotExist' was not found on your system or is not supported by Cypress. - -Cypress supports the following browsers: -- chrome -- chromium -- edge -- electron -- firefox - -You can also use a custom browser: https://on.cypress.io/customize-browsers - -Available browsers found on your system are: -- browser1 -- browser2 -- browser3 -` - exports['e2e record misconfiguration errors and exits when no specs found 1'] = ` Can't run because no spec files were found. @@ -1201,15 +955,22 @@ StatusCodeError: 422 ` -exports['e2e record api interaction errors create run 500 warns but proceeds when grouping without parallelization 1'] = ` -Warning: We encountered an error talking to our servers. +exports['e2e record api interaction errors create run 500 does not proceed and exits with error when parallelizing 1'] = ` +We encountered an unexpected error talking to our servers. -This run will not be recorded. +Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. -This error will not alter the exit code. +The --group flag you passed was: foo +The --ciBuildId flag you passed was: ciBuildId123 + +The server's response was: StatusCodeError: 500 - "Internal Server Error" +` + +exports['e2e record api interaction errors create instance 500 does not proceed and exits with error when parallelizing and creating instance 1'] = ` + ==================================================================================================== (Run Starting) @@ -1219,94 +980,18 @@ StatusCodeError: 500 - "Internal Server Error" │ Browser: FooBrowser 88 │ │ Specs: 1 found (record_pass_spec.js) │ │ Searched: cypress/integration/record_pass* │ + │ Params: Tag: nightly, Group: foo, Parallel: true │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ +We encountered an unexpected error talking to our servers. -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: record_pass_spec.js (1 of 1) +Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. +The --group flag you passed was: foo +The --ciBuildId flag you passed was: ciBuildId123 - record pass - ✓ passes - - is pending - - - 1 passing - 1 pending - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 2 │ - │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: true │ - │ Duration: X seconds │ - │ Spec Ran: record_pass_spec.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass_spec.js/yay it passes.png (400x1022) - - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass_spec.js XX:XX 2 1 - 1 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - - - -` - -exports['e2e record api interaction errors create run 500 does not proceed and exits with error when parallelizing 1'] = ` -We encountered an unexpected error talking to our servers. - -Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. - -The --group flag you passed was: foo -The --ciBuildId flag you passed was: ciBuildId123 - -The server's response was: - -StatusCodeError: 500 - "Internal Server Error" - -` - -exports['e2e record api interaction errors create instance 500 does not proceed and exits with error when parallelizing and creating instance 1'] = ` - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass_spec.js) │ - │ Searched: cypress/integration/record_pass* │ - │ Params: Tag: nightly, Group: foo, Parallel: true │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - -We encountered an unexpected error talking to our servers. - -Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. - -The --group flag you passed was: foo -The --ciBuildId flag you passed was: ciBuildId123 - -The server's response was: +The server's response was: StatusCodeError: 500 - "Internal Server Error" @@ -2185,13 +1870,8 @@ exports['e2e record passing passes 2'] = [ }, "tests": [ { - "testId": "r3", - "title": [ - "record fails", - "fails 1" - ], + "clientId": "r3", "state": "failed", - "body": "function () {}", "displayError": "Error: foo\n\nBecause this error occurred during a `before each` hook we are skipping the remaining tests in the current suite: `record fails`\n [stack trace lines]", "attempts": [ { @@ -2228,13 +1908,8 @@ exports['e2e record passing passes 2'] = [ ] }, { - "testId": "r4", - "title": [ - "record fails", - "is skipped" - ], + "clientId": "r4", "state": "skipped", - "body": "function () {}", "displayError": null, "attempts": [ { @@ -2249,18 +1924,8 @@ exports['e2e record passing passes 2'] = [ ] } ], - "error": null, + "exception": null, "video": true, - "hooks": [ - { - "hookId": "h1", - "hookName": "before each", - "title": [ - "\"before each\" hook" - ], - "body": "function () {\n throw new Error('foo');\n }" - } - ], "screenshots": [ { "screenshotId": "some-random-id", @@ -2272,7 +1937,6 @@ exports['e2e record passing passes 2'] = [ "width": 1280 } ], - "cypressConfig": {}, "reporterStats": { "suites": 1, "tests": 1, @@ -2298,13 +1962,8 @@ exports['e2e record passing passes 2'] = [ }, "tests": [ { - "testId": "r3", - "title": [ - "record pass", - "passes" - ], + "clientId": "r3", "state": "passed", - "body": "function () {\n cy.visit('/scrollable.html');\n cy.viewport(400, 400);\n cy.get('#box');\n cy.screenshot('yay it passes');\n }", "displayError": null, "attempts": [ { @@ -2325,13 +1984,8 @@ exports['e2e record passing passes 2'] = [ ] }, { - "testId": "r4", - "title": [ - "record pass", - "is pending" - ], + "clientId": "r4", "state": "pending", - "body": "", "displayError": null, "attempts": [ { @@ -2346,9 +2000,8 @@ exports['e2e record passing passes 2'] = [ ] } ], - "error": null, + "exception": null, "video": true, - "hooks": [], "screenshots": [ { "screenshotId": "some-random-id", @@ -2360,7 +2013,6 @@ exports['e2e record passing passes 2'] = [ "width": 400 } ], - "cypressConfig": {}, "reporterStats": { "suites": 1, "tests": 2, @@ -2386,12 +2038,8 @@ exports['e2e record passing passes 2'] = [ }, "tests": [ { - "testId": "r2", - "title": [ - "An uncaught error was detected outside of a test" - ], + "clientId": "r2", "state": "failed", - "body": "() => {\n throw err;\n }", "displayError": "Error: The following error originated from your test code, not from Cypress.\n\n > instantly fails\n\nWhen Cypress detects uncaught errors originating from your test code it will automatically fail the current test.\n\nCypress could not associate this error to any specific test.\n\nWe dynamically generated a new test to display this failure.\n [stack trace lines]", "attempts": [ { @@ -2425,9 +2073,8 @@ exports['e2e record passing passes 2'] = [ ] } ], - "error": null, + "exception": null, "video": true, - "hooks": [], "screenshots": [ { "screenshotId": "some-random-id", @@ -2439,7 +2086,6 @@ exports['e2e record passing passes 2'] = [ "width": 1280 } ], - "cypressConfig": {}, "reporterStats": { "suites": 0, "tests": 1, @@ -2532,3 +2178,356 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing ` + +exports['e2e record api interaction errors create run 500 errors and exits 1'] = ` +We encountered an unexpected error talking to our servers. + +The server's response was: + +StatusCodeError: 500 - "Internal Server Error" + +` + +exports['e2e record api interaction errors create run 500 when grouping without parallelization errors and exits 1'] = ` +We encountered an unexpected error talking to our servers. + +The --group flag you passed was: foo +The --ciBuildId flag you passed was: ciBuildId123 + +The server's response was: + +StatusCodeError: 500 - "Internal Server Error" + +` + +exports['e2e record api interaction errors create instance 500 without parallelization - does not proceed 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 2 found (a_record.spec.js, b_record.spec.js) │ + │ Searched: cypress/integration/*_record.spec.js │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + +We encountered an unexpected error talking to our servers. + +The server's response was: + +StatusCodeError: 500 - "Internal Server Error" + +` + +exports['e2e record api interaction errors create instance errors and exits on createInstance error 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 2 found (a_record_instantfail.spec.js, config_record_spec.js) │ + │ Searched: cypress/integration/*_record_* │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + +We encountered an unexpected error talking to our servers. + +The server's response was: + +StatusCodeError: 500 - "Internal Server Error" + +` + +exports['e2e record api interaction errors postInstanceTests without parallelization errors and exits 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 2 found (a_record.spec.js, b_record.spec.js) │ + │ Searched: cypress/integration/*_record.spec* │ + │ Params: Tag: false, Group: foo, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: a_record.spec.js (1 of 2) + Estimated: 8 seconds +We encountered an unexpected error talking to our servers. + +The --group flag you passed was: foo +The --ciBuildId flag you passed was: 1 + +The server's response was: + +StatusCodeError: 500 - "Internal Server Error" + +` + +exports['e2e record api interaction errors postInstanceTests with parallelization errors and exits 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 2 found (a_record.spec.js, b_record.spec.js) │ + │ Searched: cypress/integration/*_record.spec.js │ + │ Params: Tag: false, Group: foo, Parallel: true │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: a_record.spec.js (1 of 2) + Estimated: 8 seconds +We encountered an unexpected error talking to our servers. + +Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. + +The --group flag you passed was: foo +The --ciBuildId flag you passed was: ciBuildId123 + +The server's response was: + +StatusCodeError: 500 - "Internal Server Error" + +` + +exports['e2e record api interaction errors postInstanceResults errors and exits in serial 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (record_pass_spec.js) │ + │ Searched: cypress/integration/record_pass* │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: record_pass_spec.js (1 of 1) + Estimated: 8 seconds + + + record pass + ✓ passes + - is pending + + + 1 passing + 1 pending + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 1 │ + │ Skipped: 0 │ + │ Screenshots: 1 │ + │ Video: true │ + │ Duration: X seconds │ + │ Estimated: 8 seconds │ + │ Spec Ran: record_pass_spec.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Screenshots) + + - /XXX/XXX/XXX/cypress/screenshots/record_pass_spec.js/yay it passes.png (400x1022) + + + (Uploading Results) + +We encountered an unexpected error talking to our servers. + +The server's response was: + +StatusCodeError: 500 - "Internal Server Error" + +` + +exports['e2e record api skips specs records tests and exits without executing 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 2 found (a_record_instantfail.spec.js, b_record.spec.js) │ + │ Searched: cypress/integration/a_record_instantfail.spec.js, cypress/integration/b_record.spe │ + │ c.js │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: a_record_instantfail.spec.js (1 of 2) + Estimated: 8 seconds + + This spec and its tests were skipped because the run has been canceled. + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: b_record.spec.js (2 of 2) + Estimated: 8 seconds + + + b spec + ✓ b test + + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: 8 seconds │ + │ Spec Ran: b_record.spec.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Uploading Results) + + - Nothing to Upload + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ - a_record_instantfail.spec.js SKIPPED - - - - - │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✔ b_record.spec.js XX:XX 1 1 - - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + - The run was canceled XX:XX 1 1 - - - + + +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 + + + Exiting with non-zero exit code because the run was canceled. + +` + +exports['e2e record api skips specs records tests and exits without executing in parallel 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 2 found (a_record_instantfail.spec.js, b_record.spec.js) │ + │ Searched: cypress/integration/a_record_instantfail.spec.js, cypress/integration/b_record.spe │ + │ c.js │ + │ Params: Tag: false, Group: abc, Parallel: true │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: a_record_instantfail.spec.js (1 of 2) + Estimated: 8 seconds + + This spec and its tests were skipped because the run has been canceled. + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: b_record.spec.js (2 of 2) + Estimated: 8 seconds + + + b spec + ✓ b test + + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: 8 seconds │ + │ Spec Ran: b_record.spec.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Uploading Results) + + - Nothing to Upload + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ - a_record_instantfail.spec.js SKIPPED - - - - - │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✔ b_record.spec.js XX:XX 1 1 - - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + - The run was canceled XX:XX 1 1 - - - + + +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 + + + Exiting with non-zero exit code because the run was canceled. + +` diff --git a/packages/server/lib/api.js b/packages/server/lib/api.js index 407ff2a07224..2339c28bb184 100644 --- a/packages/server/lib/api.js +++ b/packages/server/lib/api.js @@ -22,6 +22,11 @@ let DELAYS = [ TWO_MINUTES, ] +const runnerCapabilities = { + 'dynamicSpecsInSerialMode': true, + 'skipSpecAction': true, +} + let responseCache = {} intervals = process.env.API_RETRY_INTERVALS @@ -199,19 +204,22 @@ module.exports = { }, createRun (options = {}) { - const body = _.pick(options, [ - 'ci', - 'specs', - 'commit', - 'group', - 'platform', - 'parallel', - 'ciBuildId', - 'projectId', - 'recordKey', - 'specPattern', - 'tags', - ]) + const body = { + ..._.pick(options, [ + 'ci', + 'specs', + 'commit', + 'group', + 'platform', + 'parallel', + 'ciBuildId', + 'projectId', + 'recordKey', + 'specPattern', + 'tags', + ]), + runnerCapabilities, + } return rp.post({ body, @@ -249,6 +257,22 @@ module.exports = { .catch(tagError) }, + postInstanceTests (options = {}) { + const { instanceId, ...body } = options + + return rp.post({ + url: apiRoutes.instanceTests(instanceId), + json: true, + timeout: SIXTY_SECONDS, + headers: { + 'x-route-version': '1', + }, + body, + }) + .catch(errors.StatusCodeError, formatResponseBody) + .catch(tagError) + }, + updateInstanceStdout (options = {}) { return rp.put({ url: apiRoutes.instanceStdout(options.instanceId), @@ -262,23 +286,20 @@ module.exports = { .catch(tagError) }, - updateInstance (options = {}) { - return rp.put({ - url: apiRoutes.instance(options.instanceId), + postInstanceResults (options = {}) { + return rp.post({ + url: apiRoutes.instanceResults(options.instanceId), json: true, timeout: options.timeout != null ? options.timeout : SIXTY_SECONDS, headers: { - 'x-route-version': '3', + 'x-route-version': '1', }, body: _.pick(options, [ 'stats', 'tests', - 'error', + 'exception', 'video', - 'hooks', - 'stdout', 'screenshots', - 'cypressConfig', 'reporterStats', ]), }) diff --git a/packages/server/lib/config.js b/packages/server/lib/config.js index ec3e49693c15..a95efe5bb286 100644 --- a/packages/server/lib/config.js +++ b/packages/server/lib/config.js @@ -755,6 +755,16 @@ module.exports = { , {}) }, + getResolvedRuntimeConfig (config, runtimeConfig) { + const resolvedRuntimeFields = _.mapValues(runtimeConfig, (v) => ({ value: v, from: 'runtime' })) + + return { + ...config, + ...runtimeConfig, + resolved: { ...config.resolved, ...resolvedRuntimeFields }, + } + }, + getNameFromRoot (root = '') { return path.basename(root) }, diff --git a/packages/server/lib/cypress.js b/packages/server/lib/cypress.js index 83d22d848c49..37fda1edee37 100644 --- a/packages/server/lib/cypress.js +++ b/packages/server/lib/cypress.js @@ -13,6 +13,7 @@ const R = require('ramda') const Promise = require('bluebird') const debug = require('debug')('cypress:server:cypress') const argsUtils = require('./util/args') +const chalk = require('chalk') const warning = (code, args) => { return require('./errors').warning(code, args) @@ -283,7 +284,20 @@ module.exports = { // run headlessly and exit // with num of totalFailed return this.runElectron(mode, options) - .get('totalFailed') + .then((results) => { + if (results.runs) { + const isCanceled = results.runs.filter((run) => run.skippedSpec).length + + if (isCanceled) { + // eslint-disable-next-line no-console + console.log(chalk.magenta('\n Exiting with non-zero exit code because the run was canceled.')) + + return 1 + } + } + + return results.totalFailed + }) .then(exit) .catch(exitErr) diff --git a/packages/server/lib/errors.js b/packages/server/lib/errors.js index 8578134e39da..4317572dff2d 100644 --- a/packages/server/lib/errors.js +++ b/packages/server/lib/errors.js @@ -166,6 +166,8 @@ const getMsgByType = function (type, arg1 = {}, arg2, arg3) { return `Timed out waiting for the browser to connect. ${arg1}` case 'TESTS_DID_NOT_START_FAILED': return 'The browser never connected. Something is wrong. The tests cannot run. Aborting...' + case 'DASHBOARD_CANCEL_SKIPPED_SPEC': + return '\n This spec and its tests were skipped because the run has been canceled.' case 'DASHBOARD_API_RESPONSE_FAILED_RETRYING': return stripIndent`\ We encountered an unexpected error talking to our servers. @@ -190,6 +192,19 @@ const getMsgByType = function (type, arg1 = {}, arg2, arg3) { The server's response was: + ${arg1.response}` + + case 'DASHBOARD_CANNOT_PROCEED_IN_SERIAL': + return stripIndent`\ + We encountered an unexpected error talking to our servers. + + ${displayFlags(arg1.flags, { + group: '--group', + ciBuildId: '--ciBuildId', + })} + + The server's response was: + ${arg1.response}` case 'DASHBOARD_UNKNOWN_INVALID_REQUEST': return stripIndent`\ diff --git a/packages/server/lib/modes/record.js b/packages/server/lib/modes/record.js index 5594568bed09..1142dda8312d 100644 --- a/packages/server/lib/modes/record.js +++ b/packages/server/lib/modes/record.js @@ -11,12 +11,14 @@ const logger = require('../logger') const errors = require('../errors') const capture = require('../capture') const upload = require('../upload') +const Config = require('../config') const env = require('../util/env') const keys = require('../util/keys') const terminal = require('../util/terminal') const humanTime = require('../util/human_time') const ciProvider = require('../util/ci_provider') const settings = require('../util/settings') +const testsUtils = require('../util/tests_utils') const onBeforeRetry = (details) => { return errors.warning( @@ -60,6 +62,23 @@ const warnIfProjectIdButNoRecordOption = (projectId, options) => { } } +const throwDashboardCannotProceed = ({ parallel, ciBuildId, group, err }) => { + const errMsg = parallel ? 'DASHBOARD_CANNOT_PROCEED_IN_PARALLEL' : 'DASHBOARD_CANNOT_PROCEED_IN_SERIAL' + + const errToThrow = errors.get(errMsg, { + response: err, + flags: { + group, + ciBuildId, + }, + }) + + // tells error handler to exit immediately without running anymore specs + errToThrow.isFatalApiErr = true + + throw errToThrow +} + const throwIfIndeterminateCiBuildId = (ciBuildId, parallel, group) => { if ((!ciBuildId && !ciProvider.provider()) && (parallel || group)) { errors.throw( @@ -190,29 +209,33 @@ const updateInstanceStdout = (options = {}) => { }).finally(capture.restore) } -const updateInstance = (options = {}) => { +const postInstanceResults = (options = {}) => { const { instanceId, results, group, parallel, ciBuildId } = options - let { stats, tests, hooks, video, screenshots, reporterStats, error } = results + let { stats, tests, video, screenshots, reporterStats, error } = results video = Boolean(video) - const cypressConfig = options.config // get rid of the path property screenshots = _.map(screenshots, (screenshot) => { return _.omit(screenshot, 'path') }) + tests = tests && _.map(tests, (test) => { + return _.omit({ + clientId: test.testId, + ...test, + }, 'title', 'body', 'testId') + }) + const makeRequest = () => { - return api.updateInstance({ + return api.postInstanceResults({ + instanceId, stats, tests, - error, + exception: error, video, - hooks, - instanceId, - screenshots, reporterStats, - cypressConfig, + screenshots, }) } @@ -222,25 +245,7 @@ const updateInstance = (options = {}) => { stack: err.stack, }) - if (parallel) { - return errors.throw('DASHBOARD_CANNOT_PROCEED_IN_PARALLEL', { - response: err, - flags: { - group, - ciBuildId, - }, - }) - } - - errors.warning('DASHBOARD_CANNOT_CREATE_RUN_OR_INSTANCE', err) - - // dont log exceptions if we have a 503 status code - if (err.statusCode !== 503) { - return logException(err) - .return(null) - } - - return null + throwDashboardCannotProceed({ parallel, ciBuildId, group, err }) }) } @@ -524,23 +529,7 @@ const createRun = Promise.method((options = {}) => { } } default: - if (parallel) { - return errors.throw('DASHBOARD_CANNOT_PROCEED_IN_PARALLEL', { - response: err, - flags: { - group, - ciBuildId, - }, - }) - } - - // warn the user that assets will be not recorded - errors.warning('DASHBOARD_CANNOT_CREATE_RUN_OR_INSTANCE', err) - - // report on this exception - // and return null - return logException(err) - .return(null) + throwDashboardCannotProceed({ parallel, ciBuildId, group, err }) } }) }) @@ -566,30 +555,41 @@ const createInstance = (options = {}) => { stack: err.stack, }) - if (parallel) { - return errors.throw('DASHBOARD_CANNOT_PROCEED_IN_PARALLEL', { - response: err, - flags: { - group, - ciBuildId, - }, - }) - } - - errors.warning('DASHBOARD_CANNOT_CREATE_RUN_OR_INSTANCE', err) + throwDashboardCannotProceed({ + err, + group, + ciBuildId, + parallel, + }) + }) +} - // dont log exceptions if we have a 503 status code - if (err.statusCode !== 503) { - return logException(err) - .return(null) - } +const _postInstanceTests = ({ + instanceId, + config, + tests, + hooks, + parallel, + ciBuildId, + group, +}) => { + const makeRequest = () => { + return api.postInstanceTests({ + instanceId, + config, + tests, + hooks, + }) + } - return null + return api.retryWithBackoff(makeRequest, { onBeforeRetry }) + .catch((err) => { + throwDashboardCannotProceed({ parallel, ciBuildId, group, err }) }) } const createRunAndRecordSpecs = (options = {}) => { - const { specPattern, specs, sys, browser, projectId, projectRoot, runAllSpecs, parallel, ciBuildId, group } = options + const { specPattern, specs, sys, browser, projectId, config, projectRoot, runAllSpecs, parallel, ciBuildId, group, project, onError } = options const recordKey = options.key // we want to normalize this to an array to send to API @@ -636,8 +636,7 @@ const createRunAndRecordSpecs = (options = {}) => { let instanceId = null const beforeSpecRun = (spec) => { - debug('before spec run %o', { spec }) - + project.setOnTestsReceived(onTestsReceived) capture.restore() captured = capture.stdout() @@ -653,7 +652,6 @@ const createRunAndRecordSpecs = (options = {}) => { machineId, }) .then((resp = {}) => { - resp = resp || {} instanceId = resp.instanceId // pull off only what we need @@ -670,7 +668,7 @@ const createRunAndRecordSpecs = (options = {}) => { const afterSpecRun = (spec, results, config) => { // dont do anything if we failed to // create the instance - if (!instanceId) { + if (!instanceId || results.skippedSpec) { return } @@ -686,7 +684,7 @@ const createRunAndRecordSpecs = (options = {}) => { // eslint-disable-next-line no-console console.log('') - return updateInstance({ + return postInstanceResults({ group, config, results, @@ -720,9 +718,89 @@ const createRunAndRecordSpecs = (options = {}) => { }) } + const onTestsReceived = (async (runnables, cb) => { + // we failed createInstance earlier, nothing to do + if (!instanceId) { + return + } + + const r = testsUtils.flattenSuiteIntoRunnables(runnables) + const runtimeConfig = runnables.runtimeConfig + const resolvedRuntimeConfig = Config.getResolvedRuntimeConfig(config, runtimeConfig) + + const tests = _.chain(r[0]) + .uniqBy('id') + .map((v) => { + if (v.originalTitle) { + v._titlePath.splice(-1, 1, v.originalTitle) + } + + return _.pick({ + ...v, + clientId: v.id, + config: v._testConfig || null, + title: v._titlePath, + hookIds: v.hooks.map((hook) => hook.hookId), + }, + 'clientId', 'body', 'title', 'config', 'hookIds') + }) + .value() + + const hooks = _.chain(r[1]) + .uniqBy('hookId') + .map((v) => { + return _.pick({ + ...v, + clientId: v.hookId, + title: [v.title], + type: v.hookName, + }, + 'clientId', + 'type', + 'title', + 'body') + }) + .value() + + const responseDidFail = {} + const response = await _postInstanceTests({ + instanceId, + config: resolvedRuntimeConfig, + tests, + hooks, + parallel, + ciBuildId, + group, + }) + .catch((err) => { + onError(err) + + return responseDidFail + }) + + if (response === responseDidFail) { + // dont call the cb, let the browser hang until it's killed + return + } + + if (_.some(response.actions, { type: 'SPEC', action: 'SKIP' })) { + errors.warning('DASHBOARD_CANCEL_SKIPPED_SPEC') + + // set a property on the response so the browser runner + // knows not to start executing tests + project.emit('end', { skippedSpec: true, stats: {} }) + + // dont call the cb, let the browser hang until it's killed + return + } + + return cb(response) + }) + return runAllSpecs({ runUrl, parallel, + onTestsReceived, beforeSpecRun, afterSpecRun, }) @@ -735,7 +813,9 @@ module.exports = { createInstance, - updateInstance, + postInstanceResults, + + _postInstanceTests, updateInstanceStdout, diff --git a/packages/server/lib/modes/run.js b/packages/server/lib/modes/run.js index 254c9d4516af..e2b43c5d1da2 100644 --- a/packages/server/lib/modes/run.js +++ b/packages/server/lib/modes/run.js @@ -44,7 +44,7 @@ const gray = (val) => { } const colorIf = function (val, c) { - if (val === 0) { + if (val === 0 || val == null) { val = '-' c = 'gray' } @@ -82,10 +82,16 @@ const formatBrowser = (browser) => { const formatFooterSummary = (results) => { const { totalFailed, runs } = results + const isCanceled = _.some(results.runs, { skippedSpec: true }) + // pass or fail color - const c = totalFailed ? 'red' : 'green' + const c = isCanceled ? 'magenta' : totalFailed ? 'red' : 'green' const phrase = (() => { + if (isCanceled) { + return 'The run was canceled' + } + // if we have any specs failing... if (!totalFailed) { return 'All specs passed!' @@ -100,7 +106,7 @@ const formatFooterSummary = (results) => { })() return [ - formatSymbolSummary(totalFailed), + isCanceled ? '-' : formatSymbolSummary(totalFailed), color(phrase, c), gray(duration.format(results.totalDuration)), colorIf(results.totalTests, 'reset'), @@ -339,11 +345,21 @@ const renderSummaryTable = (runUrl) => { _.each(runs, (run) => { const { spec, stats } = run - const ms = duration.format(stats.wallClockDuration) + const ms = duration.format(stats.wallClockDuration || 0) + + const formattedSpec = formatPath(spec.name, getWidth(table2, 1)) + + if (run.skippedSpec) { + return table2.push([ + '-', + formattedSpec, color('SKIPPED', 'gray'), + '-', '-', '-', '-', '-', + ]) + } return table2.push([ formatSymbolSummary(stats.failures), - formatPath(spec.name, getWidth(table2, 1)), + formattedSpec, color(ms, 'gray'), colorIf(stats.tests, 'reset'), colorIf(stats.passes, 'green'), @@ -381,28 +397,27 @@ const renderSummaryTable = (runUrl) => { } const iterateThroughSpecs = function (options = {}) { - const { specs, runEachSpec, parallel, beforeSpecRun, afterSpecRun, config } = options + const { specs, runEachSpec, beforeSpecRun, afterSpecRun, config } = options const serial = () => { return Promise.mapSeries(specs, runEachSpec) } - const serialWithRecord = () => { - return Promise - .mapSeries(specs, (spec, index, length) => { - return beforeSpecRun(spec) - .then(({ estimated }) => { - return runEachSpec(spec, index, length, estimated) - }) - .tap((results) => { - return afterSpecRun(spec, results, config) - }) - }) - } - - const parallelWithRecord = (runs) => { + const ranSpecs = [] + const parallelAndSerialWithRecord = (runs) => { return beforeSpecRun() - .then(({ spec, claimedInstances, totalInstances, estimated }) => { + .then(({ spec, claimedInstances, totalInstances, estimated, shouldFallbackToOfflineOrder }) => { + // if (!parallel) { + // // NOTE: if we receive the old API which always sends {spec: null}, + // // that would instantly end the run with a 0 exit code if we act like parallel mode. + // // so instead we check length of ran specs just to make sure we have run all the specs. + // // However, this means the api can't end a run early for us without some other logic being added. + + // if (shouldFallbackToOfflineOrder) { + // spec = _.without(specs, ...ranSpecs)[0]?.relative + // } + // } + // no more specs to run? if (!spec) { // then we're done! @@ -413,6 +428,7 @@ const iterateThroughSpecs = function (options = {}) { // our specs array since the API sends us // the relative name spec = _.find(specs, { relative: spec }) + ranSpecs.push(spec) return runEachSpec( spec, @@ -426,22 +442,21 @@ const iterateThroughSpecs = function (options = {}) { return afterSpecRun(spec, results, config) }) .then(() => { + // // no need to make an extra request if we know we've run all the specs + // if (!parallel && ranSpecs.length === specs.length) { + // return runs + // } + // recurse - return parallelWithRecord(runs) + return parallelAndSerialWithRecord(runs) }) }) } - if (parallel) { + if (beforeSpecRun) { // if we are running in parallel // then ask the server for the next spec - return parallelWithRecord([]) - } - - if (beforeSpecRun) { - // else iterate serialially and record - // the results of each spec - return serialWithRecord() + return parallelAndSerialWithRecord([]) } // else iterate in serial @@ -967,7 +982,7 @@ module.exports = { }, listenForProjectEnd (project, exit) { - return new Promise((resolve) => { + return new Promise((resolve, reject) => { if (exit === false) { resolve = () => { console.log('not exiting due to options.exit being false') @@ -975,6 +990,10 @@ module.exports = { } const onEarlyExit = function (err) { + if (err.isFatalApiErr) { + return reject(err) + } + console.log('') errors.log(err) @@ -1154,6 +1173,9 @@ module.exports = { reporterStats: null, }) + // dashboard told us to skip this spec + const skippedSpec = results.skippedSpec + if (startedVideoCapture) { results.video = videoName } @@ -1196,11 +1218,11 @@ module.exports = { const hasFailingTests = _.get(stats, 'failures') > 0 // we should upload the video if we upload on passes (by default) // or if we have any failures and have started the video - const shouldUploadVideo = videoUploadOnPasses === true || Boolean((startedVideoCapture && hasFailingTests)) + const shouldUploadVideo = !skippedSpec && videoUploadOnPasses === true || Boolean((startedVideoCapture && hasFailingTests)) results.shouldUploadVideo = shouldUploadVideo - if (!quiet) { + if (!quiet && !skippedSpec) { this.displayResults(results, estimated) if (screenshots && screenshots.length) { this.displayScreenshots(screenshots) @@ -1215,7 +1237,7 @@ module.exports = { await openProject.closeBrowser() } - if (videoExists && endVideoCapture && !videoCaptureFailed) { + if (videoExists && !skippedSpec && endVideoCapture && !videoCaptureFailed) { const ffmpegChaptersConfig = videoCapture.generateFfmpegChaptersConfig(results.tests) await this.postProcessRecording( @@ -1599,19 +1621,22 @@ module.exports = { const { projectName } = config return recordMode.createRunAndRecordSpecs({ + tag, key, sys, specs, group, - tag, + config, browser, parallel, ciBuildId, + project, projectId, projectRoot, projectName, specPattern, runAllSpecs, + onError, }) } diff --git a/packages/server/lib/open_project.js b/packages/server/lib/open_project.js index 756df9a95ec2..2cb93227f3b6 100644 --- a/packages/server/lib/open_project.js +++ b/packages/server/lib/open_project.js @@ -321,7 +321,7 @@ const moduleFactory = () => { options.configFile = args.configFile } - options = _.extend({}, args.config, options) + options = _.extend({}, args.config, options, { args }) // open the project and return // the config for the project instance diff --git a/packages/server/lib/project-base.ts b/packages/server/lib/project-base.ts index 37870b828c3e..5252ff909e6f 100644 --- a/packages/server/lib/project-base.ts +++ b/packages/server/lib/project-base.ts @@ -58,6 +58,7 @@ export class ProjectBase extends EE { protected _cfg?: Cfg protected _server?: TServer protected _automation?: Automation + private _recordTests = null public browser: any @@ -93,6 +94,10 @@ export class ProjectBase extends EE { throw new Error('Project#projectType must be defined') } + setOnTestsReceived (fn) { + this._recordTests = fn + } + get server () { return this.ensureProp(this._server, 'open') } @@ -348,12 +353,22 @@ export class ProjectBase extends EE { this.emit('socket:connected', id) }, - onSetRunnables (runnables) { + onTestsReceivedAndMaybeRecord: async (runnables, cb) => { debug('received runnables %o', runnables) if (reporter != null) { reporter.setRunnables(runnables) } + + if (this._recordTests) { + await this._recordTests(runnables, cb) + + this._recordTests = null + + return + } + + cb() }, onMocha: (event, runnable) => { diff --git a/packages/server/lib/socket-base.ts b/packages/server/lib/socket-base.ts index 998777701341..05387f221d30 100644 --- a/packages/server/lib/socket-base.ts +++ b/packages/server/lib/socket-base.ts @@ -140,7 +140,7 @@ export class SocketBase { _.defaults(options, { socketId: null, onResetServerState () {}, - onSetRunnables () {}, + onTestsReceivedAndMaybeRecord () {}, onMocha () {}, onConnect () {}, onRequest () {}, @@ -284,10 +284,8 @@ export class SocketBase { return options.onConnect(socketId, socket) }) - socket.on('set:runnables', (runnables, cb) => { - options.onSetRunnables(runnables) - - return cb() + socket.on('set:runnables:and:maybe:record:tests', async (runnables, cb) => { + return options.onTestsReceivedAndMaybeRecord(runnables, cb) }) socket.on('mocha', (...args: unknown[]) => { diff --git a/packages/server/lib/util/routes.js b/packages/server/lib/util/routes.js index d6babc838219..0281c565e56e 100644 --- a/packages/server/lib/util/routes.js +++ b/packages/server/lib/util/routes.js @@ -50,7 +50,8 @@ const apiRoutes = makeRoutes(apiUrl, { ping: 'ping', runs: 'runs', instances: 'runs/:id/instances', - instance: 'instances/:id', + instanceTests: 'instances/:id/tests', + instanceResults: 'instances/:id/results', instanceStdout: 'instances/:id/stdout', orgs: 'organizations', projects: 'projects', diff --git a/packages/server/lib/util/tests_utils.ts b/packages/server/lib/util/tests_utils.ts new file mode 100644 index 000000000000..3242b81a28b4 --- /dev/null +++ b/packages/server/lib/util/tests_utils.ts @@ -0,0 +1,20 @@ +import _ from 'lodash' + +export const flattenSuiteIntoRunnables = (suite, tests = [], hooks = []) => { + if (_.isArray(suite)) { + return _.map(suite, (s) => flattenSuiteIntoRunnables(s)) + .reduce( + (arr1, arr2) => [arr1[0].concat(arr2[0]), arr1[1].concat(arr2[1])], + [tests, hooks], + ) + } + + tests = tests.concat(suite.tests) + hooks = hooks.concat(suite.hooks) + + if (suite.suites.length) { + return flattenSuiteIntoRunnables(suite.suites, tests, hooks) + } + + return [tests, hooks] +} diff --git a/packages/server/package.json b/packages/server/package.json index 28ca4b704730..1040af85290a 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -127,7 +127,7 @@ "@babel/core": "7.9.0", "@babel/preset-env": "7.9.0", "@cypress/debugging-proxy": "2.0.1", - "@cypress/json-schemas": "5.35.1", + "@cypress/json-schemas": "5.37.3", "@cypress/sinon-chai": "2.9.1", "@ffprobe-installer/ffprobe": "1.1.0", "@packages/desktop-gui": "0.0.0-development", diff --git a/packages/server/test/e2e/7_record_spec.js b/packages/server/test/e2e/7_record_spec.js index 51d4c02c79a5..5444b2b7d273 100644 --- a/packages/server/test/e2e/7_record_spec.js +++ b/packages/server/test/e2e/7_record_spec.js @@ -1,16 +1,21 @@ const _ = require('lodash') const path = require('path') const Promise = require('bluebird') -const bodyParser = require('body-parser') const jsonSchemas = require('@cypress/json-schemas').api const snapshot = require('snap-shot-it') const e2e = require('../support/helpers/e2e').default const { fs } = require('../../lib/util/fs') const Fixtures = require('../support/helpers/fixtures') +const { + createRoutes, + setupStubbedServer, + getRequestUrls, getRequests, + postRunResponse, + postRunResponseWithWarnings, + postRunInstanceResponse, + postInstanceTestsResponse, +} = require('../support/helpers/serverStub') const { expectRunsToHaveCorrectTimings } = require('../support/helpers/resultsUtils') -const postRunResponseWithWarnings = jsonSchemas.getExample('postRunResponse')('2.2.0') -const postRunResponse = _.assign({}, postRunResponseWithWarnings, { warnings: [] }) -const postRunInstanceResponse = jsonSchemas.getExample('postRunInstanceResponse')('2.1.0') const { clearCypressJsonCache } = require('../specUtils') const e2ePath = Fixtures.projectPath('e2e') @@ -21,214 +26,13 @@ const { instanceId } = postRunInstanceResponse let requests = null -const getRequestUrls = () => { - return _.map(requests, 'url') -} - -const getSchemaErr = (tag, err, schema) => { - return { - errors: err.errors, - object: err.object, - example: err.example, - message: `${tag} should follow ${schema} schema`, - } -} - -const getResponse = function (responseSchema) { - if (_.isObject(responseSchema)) { - return responseSchema - } - - const [name, version] = responseSchema.split('@') - - return jsonSchemas.getExample(name)(version) -} - -const sendResponse = function (req, res, responseBody) { - if (_.isFunction(responseBody)) { - return responseBody(req, res) - } - - return res.json(getResponse(responseBody)) -} - -const ensureSchema = function (expectedRequestSchema, responseBody, expectedResponseSchema) { - let reqName; let reqVersion - - if (expectedRequestSchema) { - [reqName, reqVersion] = expectedRequestSchema.split('@') - } - - return function (req, res) { - const { body } = req - - try { - if (expectedRequestSchema) { - jsonSchemas.assertSchema(reqName, reqVersion)(body) - } - - res.expectedResponseSchema = expectedResponseSchema - - sendResponse(req, res, responseBody) - - const key = [req.method, req.url].join(' ') - - return requests.push({ - url: key, - body, - }) - } catch (err) { - return res.status(412).json(getSchemaErr('request', err, expectedRequestSchema)) - } - } -} - -const sendUploadUrls = function (req, res) { - const { body } = req - - let num = 0 - - const json = {} - - if (body.video) { - json.videoUploadUrl = 'http://localhost:1234/videos/video.mp4' - } - - const screenshotUploadUrls = _.map(body.screenshots, (s) => { - num += 1 - - return { - screenshotId: s.screenshotId, - uploadUrl: `http://localhost:1234/screenshots/${num}.png`, - } - }) - - json.screenshotUploadUrls = screenshotUploadUrls - - return res.json(json) -} - -const assertResponseBodySchema = function (req, res, next) { - const oldWrite = res.write - const oldEnd = res.end - - const chunks = [] - - res.write = (chunk) => { - // buffer the response, we'll really write it on end - return chunks.push(chunk) - } - - res.end = function (chunk) { - if (chunk) { - chunks.push(chunk) - } - - res.write = oldWrite - res.end = oldEnd - - if (res.expectedResponseSchema && _.inRange(res.statusCode, 200, 299)) { - const body = JSON.parse(Buffer.concat(chunks).toString('utf8')) - - const [resName, resVersion] = res.expectedResponseSchema.split('@') - - try { - jsonSchemas.assertSchema(resName, resVersion)(body) - } catch (err) { - return res.status(412).json(getSchemaErr('response', err, res.expectedResponseSchema)) - } - } - - chunks.map((chunk) => { - return res.write(chunk) - }) - - return res.end() - } - - return next() -} - -const onServer = (routes) => { - return (function (app) { - app.use(bodyParser.json()) - - app.use(assertResponseBodySchema) - - return _.each(routes, (route) => { - return app[route.method](route.url, ensureSchema( - route.req, - route.res, - route.resSchema, - )) - }) - }) -} - -const setup = (routes, settings = {}) => { - return e2e.setup({ - settings: _.extend({ - projectId: 'pid123', - videoUploadOnPasses: false, - }, settings), - servers: { - port: 1234, - onServer: onServer(routes), - }, - }) -} - -const defaultRoutes = [ - { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - resSchema: 'postRunResponse@2.2.0', - res: postRunResponse, - }, { - method: 'post', - url: '/runs/:id/instances', - req: 'postRunInstanceRequest@2.1.0', - resSchema: 'postRunInstanceResponse@2.1.0', - res: postRunInstanceResponse, - }, { - method: 'put', - url: '/instances/:id', - req: 'putInstanceRequest@3.0.0', - resSchema: 'putInstanceResponse@2.0.0', - res: sendUploadUrls, - }, { - method: 'put', - url: '/instances/:id/stdout', - req: 'putInstanceStdoutRequest@1.0.0', - res (req, res) { - return res.sendStatus(200) - }, - }, { - method: 'put', - url: '/videos/:name', - res (req, res) { - return Promise.delay(500) - .then(() => { - return res.sendStatus(200) - }) - }, - }, { - method: 'put', - url: '/screenshots/:name', - res (req, res) { - return res.sendStatus(200) - }, - }, -] - describe('e2e record', () => { beforeEach(() => { - return requests = [] + requests = getRequests() }) context('passing', () => { - setup(defaultRoutes) + setupStubbedServer(createRoutes()) it('passes', async function () { const { stdout } = await e2e.exec(this, { @@ -246,50 +50,41 @@ describe('e2e record', () => { const urls = getRequestUrls() - // first create run request - expect(urls[0]).to.eq('POST /runs') + const instanceReqs = urls.slice(0, 22) - // grab the first set of 4 - const firstInstanceSet = urls.slice(1, 5) + expect(instanceReqs).to.deep.eq([ + // first create run request + 'POST /runs', - expect(firstInstanceSet).to.deep.eq([ + // spec 1 `POST /runs/${runId}/instances`, - `PUT /instances/${instanceId}`, + // no instances/:id/tests becuase spec failed during eval + `POST /instances/${instanceId}/results`, 'PUT /videos/video.mp4', `PUT /instances/${instanceId}/stdout`, - ]) - - // grab the second set of 5 - const secondInstanceSet = urls.slice(5, 10) - console.log(secondInstanceSet) - expect(secondInstanceSet).to.have.members([ + // spec 2 `POST /runs/${runId}/instances`, - `PUT /instances/${instanceId}`, - 'PUT /videos/video.mp4', + `POST /instances/${instanceId}/tests`, + `POST /instances/${instanceId}/results`, 'PUT /screenshots/1.png', + 'PUT /videos/video.mp4', `PUT /instances/${instanceId}/stdout`, - ]) - // grab the third set of 5 - const thirdInstanceSet = urls.slice(10, 14) - - // no video because no tests failed - expect(thirdInstanceSet).to.deep.eq([ + // spec 3 `POST /runs/${runId}/instances`, - `PUT /instances/${instanceId}`, + `POST /instances/${instanceId}/tests`, + `POST /instances/${instanceId}/results`, + // no video because no tests failed 'PUT /screenshots/1.png', `PUT /instances/${instanceId}/stdout`, - ]) - // grab the forth set of 5 - const forthInstanceSet = urls.slice(14, 19) - - expect(forthInstanceSet).to.have.members([ + // spec 4 `POST /runs/${runId}/instances`, - `PUT /instances/${instanceId}`, - 'PUT /videos/video.mp4', + `POST /instances/${instanceId}/tests`, + `POST /instances/${instanceId}/results`, 'PUT /screenshots/1.png', + 'PUT /videos/video.mp4', `PUT /instances/${instanceId}/stdout`, ]) @@ -311,19 +106,17 @@ describe('e2e record', () => { expect(firstInstance.body.groupId).to.eq(groupId) expect(firstInstance.body.machineId).to.eq(machineId) - expect(firstInstance.body.spec).to.eq( - 'cypress/integration/record_error_spec.js', - ) + expect(firstInstance.body.spec).to.eq(null) - const firstInstancePut = requests[2] + const firstInstancePostResults = requests[2] - expect(firstInstancePut.body.error).to.include('Oops...we found an error preparing this test file') - expect(firstInstancePut.body.tests).to.be.null - expect(firstInstancePut.body.hooks).to.be.null - expect(firstInstancePut.body.screenshots).to.have.length(0) - expect(firstInstancePut.body.stats.tests).to.eq(0) - expect(firstInstancePut.body.stats.failures).to.eq(1) - expect(firstInstancePut.body.stats.passes).to.eq(0) + expect(firstInstancePostResults.body.exception).to.include('Oops...we found an error preparing this test file') + expect(firstInstancePostResults.body.tests).to.be.null + expect(firstInstancePostResults.body.hooks).to.not.exist + expect(firstInstancePostResults.body.screenshots).to.have.length(0) + expect(firstInstancePostResults.body.stats.tests).to.eq(0) + expect(firstInstancePostResults.body.stats.failures).to.eq(1) + expect(firstInstancePostResults.body.stats.passes).to.eq(0) const firstInstanceStdout = requests[4] @@ -333,77 +126,91 @@ describe('e2e record', () => { expect(secondInstance.body.groupId).to.eq(groupId) expect(secondInstance.body.machineId).to.eq(machineId) - expect(secondInstance.body.spec).to.eq( - 'cypress/integration/record_fail_spec.js', - ) + expect(secondInstance.body.spec).to.eq(null) + + const secondInstancePostTests = requests[6].body - const secondInstancePut = requests[6] + expect(secondInstancePostTests.tests).length(2) + expect(secondInstancePostTests.hooks).length(1) + expect(secondInstancePostTests.config).is.an('object') - expect(secondInstancePut.body.error).to.be.null - expect(secondInstancePut.body.tests).to.have.length(2) - expect(secondInstancePut.body.hooks).to.have.length(1) - expect(secondInstancePut.body.screenshots).to.have.length(1) - expect(secondInstancePut.body.stats.tests).to.eq(2) - expect(secondInstancePut.body.stats.failures).to.eq(1) - expect(secondInstancePut.body.stats.passes).to.eq(0) - expect(secondInstancePut.body.stats.skipped).to.eq(1) + const secondInstancePostResults = requests[7] - const secondInstanceStdout = requests[9] + expect(secondInstancePostResults.body.exception).to.be.null + expect(secondInstancePostResults.body.tests).to.have.length(2) + expect(secondInstancePostResults.body.screenshots).to.have.length(1) + expect(secondInstancePostResults.body.stats.tests).to.eq(2) + expect(secondInstancePostResults.body.stats.failures).to.eq(1) + expect(secondInstancePostResults.body.stats.passes).to.eq(0) + expect(secondInstancePostResults.body.stats.skipped).to.eq(1) + expect(secondInstancePostResults.body.hooks).not.exist + expect(secondInstancePostResults.body.cypressConfig).not.exist + + const secondInstanceStdout = requests[10] expect(secondInstanceStdout.body.stdout).to.include('record_fail_spec.js') expect(secondInstanceStdout.body.stdout).not.to.include('record_error_spec.js') - const thirdInstance = requests[10] + const thirdInstance = requests[11] expect(thirdInstance.body.groupId).to.eq(groupId) expect(thirdInstance.body.machineId).to.eq(machineId) - expect(thirdInstance.body.spec).to.eq( - 'cypress/integration/record_pass_spec.js', - ) + expect(thirdInstance.body.spec).to.eq(null) + + const thirdInstancePostTests = requests[12].body - const thirdInstancePut = requests[11] + expect(thirdInstancePostTests.tests[0].config.env.foo).eq(true) + expect(thirdInstancePostTests.tests).length(2) + expect(thirdInstancePostTests.hooks).length(0) + expect(thirdInstancePostTests.config).is.an('object') - expect(thirdInstancePut.body.error).to.be.null - expect(thirdInstancePut.body.tests).to.have.length(2) - expect(thirdInstancePut.body.hooks).to.have.length(0) - expect(thirdInstancePut.body.screenshots).to.have.length(1) - expect(thirdInstancePut.body.stats.tests).to.eq(2) - expect(thirdInstancePut.body.stats.passes).to.eq(1) - expect(thirdInstancePut.body.stats.failures).to.eq(0) - expect(thirdInstancePut.body.stats.pending).to.eq(1) + const thirdInstancePostResults = requests[13] - const thirdInstanceStdout = requests[13] + expect(thirdInstancePostResults.body.exception).to.be.null + expect(thirdInstancePostResults.body.tests).to.have.length(2) + expect(thirdInstancePostResults.body.screenshots).to.have.length(1) + expect(thirdInstancePostResults.body.stats.tests).to.eq(2) + expect(thirdInstancePostResults.body.stats.passes).to.eq(1) + expect(thirdInstancePostResults.body.stats.failures).to.eq(0) + expect(thirdInstancePostResults.body.stats.pending).to.eq(1) + + const thirdInstanceStdout = requests[15] + + console.log('13') expect(thirdInstanceStdout.body.stdout).to.include('record_pass_spec.js') expect(thirdInstanceStdout.body.stdout).not.to.include('record_error_spec.js') expect(thirdInstanceStdout.body.stdout).not.to.include('record_fail_spec.js') - const fourthInstance = requests[14] + const fourthInstance = requests[16] + + console.log('14') expect(fourthInstance.body.groupId).to.eq(groupId) expect(fourthInstance.body.machineId).to.eq(machineId) - expect(fourthInstance.body.spec).to.eq( - 'cypress/integration/record_uncaught_spec.js', - ) + expect(fourthInstance.body.spec).to.eq(null) + + const fourthInstancePostResults = requests[18] + + console.log('15') - const fourthInstancePut = requests[15] + expect(fourthInstancePostResults.body.exception).to.be.null + expect(fourthInstancePostResults.body.tests).to.have.length(1) + expect(fourthInstancePostResults.body.screenshots).to.have.length(1) + expect(fourthInstancePostResults.body.stats.tests).to.eq(1) + expect(fourthInstancePostResults.body.stats.failures).to.eq(1) + expect(fourthInstancePostResults.body.stats.passes).to.eq(0) - expect(fourthInstancePut.body.error).to.be.null - expect(fourthInstancePut.body.tests).to.have.length(1) - expect(fourthInstancePut.body.hooks).to.have.length(0) - expect(fourthInstancePut.body.screenshots).to.have.length(1) - expect(fourthInstancePut.body.stats.tests).to.eq(1) - expect(fourthInstancePut.body.stats.failures).to.eq(1) - expect(fourthInstancePut.body.stats.passes).to.eq(0) + const forthInstanceStdout = requests[21] - const forthInstanceStdout = requests[18] + console.log('18') expect(forthInstanceStdout.body.stdout).to.include('record_uncaught_spec.js') expect(forthInstanceStdout.body.stdout).not.to.include('record_error_spec.js') expect(forthInstanceStdout.body.stdout).not.to.include('record_fail_spec.js') expect(forthInstanceStdout.body.stdout).not.to.include('record_pass_spec.js') - let runs = requests.filter((v) => v.body.tests).map((v) => v.body) + let runs = requests.filter((v) => v.url.match(/POST \/instances\/.*\/results/) && v.body.tests).map((v) => v.body) expectRunsToHaveCorrectTimings(runs) @@ -458,86 +265,76 @@ describe('e2e record', () => { } // replace the 1st + 2nd routes object - const routes = defaultRoutes.slice(0) - - routes[0] = { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - resSchema: 'postRunResponse@2.2.0', - res (req, res) { - let ciBuildId; let group; - - ({ group, tags, ciBuildId } = req.body) - - expect(group).to.eq('prod-e2e') - expect(tags).to.deep.eq(['nightly']) - expect(ciBuildId).to.eq('ciBuildId123') - - // if this is the first response - // give machineId a1, else b2 - if (!firstRunResponse) { - firstRunResponse = true - machineId = 'a1ad2bcf-6398-46ed-b201-2fd90b188d5f' - } else { - machineId = 'b2bd2bcf-6398-46ed-b201-2fd90b188d5f' - } - - return res.json( - _.extend({}, postRunResponse, { machineId }), - ) - }, + const routes = createRoutes({ + postRun: { + res (req, res) { + let ciBuildId; let group; - } + ({ group, tags, ciBuildId } = req.body) - routes[1] = { - method: 'post', - url: '/runs/:id/instances', - req: 'postRunInstanceRequest@2.1.0', - resSchema: 'postRunInstanceResponse@2.1.0', - res (req, res) { - let spec; + expect(group).to.eq('prod-e2e') + expect(tags).to.deep.eq(['nightly']) + expect(ciBuildId).to.eq('ciBuildId123') - ({ machineId, spec } = req.body) + // if this is the first response + // give machineId a1, else b2 + if (!firstRunResponse) { + firstRunResponse = true + machineId = 'a1ad2bcf-6398-46ed-b201-2fd90b188d5f' + } else { + machineId = 'b2bd2bcf-6398-46ed-b201-2fd90b188d5f' + } - expect(spec).to.be.null + return res.json( + _.extend({}, postRunResponse, { machineId }), + ) + }, + }, + postRunInstance: { + res (req, res) { + let spec; - const mId = machineId.slice(0, 2) + ({ machineId, spec } = req.body) - const respond = function () { - const resp = responses[mId].shift() + expect(spec).to.be.null - // if theres a spec to claim - if (resp.spec) { - claimed.push(resp) - } + const mId = machineId.slice(0, 2) - resp.claimedInstances = claimed.length - resp.totalInstances = allSpecs.length + const respond = function () { + const resp = responses[mId].shift() - jsonSchemas.assertSchema('postRunInstanceResponse', '2.1.0')(resp) + // if theres a spec to claim + if (resp.spec) { + claimed.push(resp) + } - return res.json(resp) - } + resp.claimedInstances = claimed.length + resp.totalInstances = allSpecs.length - // when the 1st machine attempts to claim its FIRST spec, we - // automatically delay it until the 2nd machine claims its FIRST - // spec so that the request URL's are deterministic - if ((mId === 'a1') && (claimed.length === 0)) { - waitUntilSecondInstanceClaims = function () { - waitUntilSecondInstanceClaims = null + jsonSchemas.assertSchema('postRunInstanceResponse', '2.1.0')(resp) - return respond() + return res.json(resp) } - } else { - respond() - return (typeof waitUntilSecondInstanceClaims === 'function' ? waitUntilSecondInstanceClaims() : undefined) - } + // when the 1st machine attempts to claim its FIRST spec, we + // automatically delay it until the 2nd machine claims its FIRST + // spec so that the request URL's are deterministic + if ((mId === 'a1') && (claimed.length === 0)) { + waitUntilSecondInstanceClaims = function () { + waitUntilSecondInstanceClaims = null + + return respond() + } + } else { + respond() + + return (typeof waitUntilSecondInstanceClaims === 'function' ? waitUntilSecondInstanceClaims() : undefined) + } + }, }, - } + }) - setup(routes) + setupStubbedServer(routes) it('passes in parallel with group', function () { this.retries(3) @@ -584,7 +381,7 @@ describe('e2e record', () => { }) context('misconfiguration', () => { - setup([]) + setupStubbedServer([]) it('errors and exits when no specs found', function () { return e2e.exec(this, { @@ -625,7 +422,7 @@ describe('e2e record', () => { }) context('recordKey', () => { - setup(defaultRoutes) + setupStubbedServer(createRoutes()) it('errors and exits without recordKey', function () { return e2e.exec(this, { @@ -678,8 +475,161 @@ describe('e2e record', () => { }) }) + context('test configuration', () => { + setupStubbedServer(createRoutes(), { + video: false, + defaultCommandTimeout: 9999, + }) + + it('config from runtime, testOptions', async function () { + await e2e.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + spec: 'config_record_spec*', + record: true, + snapshot: false, + + }) + + expect(requests[2].body.config.defaultCommandTimeout).eq(1111) + expect(requests[2].body.config.resolved.defaultCommandTimeout).deep.eq({ + value: 1111, + from: 'runtime', + }) + + expect(requests[2].body.config.pageLoadTimeout).eq(3333) + expect(requests[2].body.config.resolved.pageLoadTimeout).deep.eq({ + value: 3333, + from: 'runtime', + }) + + expect(requests[2].body.tests[0].config).deep.eq({ + defaultCommandTimeout: 1234, + env: { foo: true }, + retries: 2, + }) + + expect(requests[2].body.tests[3].title).deep.eq([ + 'record pass', + 'is skipped due to browser', + ]) + + expect(requests[2].body.tests[3].config).deep.eq({ + defaultCommandTimeout: 1234, + browser: 'edge', + }) + }) + }) + + context('record in non-parallel', () => { + describe('api reordering specs', () => { + let mockServerState + + mockServerState = setupStubbedServer(createRoutes({ + postRun: { + res (req, res) { + console.log(req.body.specs) + mockServerState.specs = req.body.specs.slice().reverse() + console.log(mockServerState.specs) + mockServerState.allSpecs = req.body.specs + res.json(postRunResponse) + }, + }, + }), { video: false }) + + it('changes spec run order', async function () { + await e2e.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + spec: 'a_record.spec.js,b_record.spec.js', + record: true, + snapshot: false, + }) + + // specs were reordered + expect(requests[2].body.tests[0].title[1]).eq('b test') + expect(requests[6].body.tests[0].title[1]).eq('a test') + }) + }) + }) + + describe('api skips specs', () => { + mockServerState = setupStubbedServer(createRoutes({ + + postInstanceTests: { + res: (req, res) => { + console.log(mockServerState.specs) + if (mockServerState.specs.length > 0) { + return res.json({ + ...postInstanceTestsResponse, + actions: [{ + type: 'SPEC', + action: 'SKIP', + }], + }) + } + + return res.json({ + ...postInstanceTestsResponse, + actions: [], + }) + }, + }, + + }), { video: false }) + + it('records tests and exits without executing', async function () { + await e2e.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + spec: 'a_record_instantfail.spec.js,b_record.spec.js', + record: true, + snapshot: true, + expectedExitCode: 1, + }) + + expect(getRequestUrls()).deep.eq([ + 'POST /runs', + 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', + 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/tests', + 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', + 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/tests', + 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/results', + 'PUT /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/stdout', + 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', + ]) + + console.log(requests[0].body.runnerCapabilities) + expect(requests[0].body).property('runnerCapabilities').deep.eq({ + 'dynamicSpecsInSerialMode': true, + 'skipSpecAction': true, + }) + }) + + it('records tests and exits without executing in parallel', async function () { + await e2e.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + spec: 'a_record_instantfail.spec.js,b_record.spec.js', + record: true, + snapshot: true, + group: 'abc', + parallel: true, + ciBuildId: 'ciBuildId123', + expectedExitCode: 1, + }) + + expect(getRequestUrls()).deep.eq([ + 'POST /runs', + 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', + 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/tests', + 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', + 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/tests', + 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/results', + 'PUT /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/stdout', + 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', + ]) + }) + }) + context('video recording', () => { - setup(defaultRoutes, { + setupStubbedServer(createRoutes(), { video: false, }) @@ -695,18 +645,15 @@ describe('e2e record', () => { context('api interaction errors', () => { describe('recordKey and projectId', () => { - const routes = [ - { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', + const routes = createRoutes({ + postRun: { res (req, res) { return res.sendStatus(401) }, }, - ] + }) - setup(routes) + setupStubbedServer(routes) it('errors and exits on 401', function () { return e2e.exec(this, { @@ -720,18 +667,15 @@ describe('e2e record', () => { }) describe('project 404', () => { - const routes = [ - { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', + const routes = createRoutes({ + postRun: { res (req, res) { return res.sendStatus(404) }, }, - ] + }) - setup(routes) + setupStubbedServer(routes) it('errors and exits', function () { return e2e.exec(this, { @@ -749,18 +693,17 @@ describe('e2e record', () => { clearCypressJsonCache() }) - const routes = [{ - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - res (req, res) { - return res.sendStatus(500) + const routes = createRoutes({ + postRun: { + res (req, res) { + return res.sendStatus(500) + }, }, - }] + }) - setup(routes) + setupStubbedServer(routes) - it('warns and does not create or update instances', function () { + it('errors and exits', function () { process.env.DISABLE_API_RETRIES = 'true' return e2e.exec(this, { @@ -768,6 +711,7 @@ describe('e2e record', () => { spec: 'record_pass*', record: true, snapshot: true, + expectedExitCode: 1, }) .then(() => { const urls = getRequestUrls() @@ -778,7 +722,7 @@ describe('e2e record', () => { }) }) - it('warns but proceeds when grouping without parallelization', function () { + it('when grouping without parallelization errors and exits', function () { process.env.DISABLE_API_RETRIES = 'true' return e2e.exec(this, { @@ -788,6 +732,7 @@ describe('e2e record', () => { record: true, snapshot: true, ciBuildId: 'ciBuildId123', + expectedExitCode: 1, }) .then(() => { const urls = getRequestUrls() @@ -823,19 +768,15 @@ describe('e2e record', () => { }) describe('create instance 500', () => { - const routes = defaultRoutes.slice(0) - - routes[1] = { - method: 'post', - url: '/runs/:id/instances', - req: 'postRunInstanceRequest@2.1.0', - resSchema: 'postRunInstanceResponse@2.1.0', - res (req, res) { - return res.sendStatus(500) + const routes = createRoutes({ + postRunInstance: { + res (req, res) { + return res.sendStatus(500) + }, }, - } + }) - setup(routes) + setupStubbedServer(_.values(routes)) it('does not proceed and exits with error when parallelizing and creating instance', function () { process.env.DISABLE_API_RETRIES = 'true' @@ -860,37 +801,49 @@ describe('e2e record', () => { ]) }) }) + + it('without parallelization - does not proceed', async function () { + process.env.DISABLE_API_RETRIES = 'true' + + await e2e.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + spec: '*_record.spec.js', + record: true, + snapshot: true, + expectedExitCode: 1, + }) + .then(() => { + const urls = getRequestUrls() + + expect(urls).to.deep.eq([ + 'POST /runs', + 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', + ]) + }) + }) }) describe('update instance 500', () => { - const routes = defaultRoutes.slice(0) - - routes[1] = { - method: 'post', - url: '/runs/:id/instances', - req: 'postRunInstanceRequest@2.1.0', - resSchema: 'postRunInstanceResponse@2.1.0', - res (req, res) { - return res.json({ - instanceId, - spec: 'cypress/integration/record_pass_spec.js', - estimatedWallClockDuration: 5000, - totalInstances: 1, - claimedInstances: 1, - }) + const routes = createRoutes({ + postRunInstance: { + res (req, res) { + return res.json({ + instanceId, + spec: 'cypress/integration/record_pass_spec.js', + estimatedWallClockDuration: 5000, + totalInstances: 1, + claimedInstances: 1, + }) + }, }, - } - - routes[2] = { - method: 'put', - url: '/instances/:id', - req: 'putInstanceRequest@3.0.0', - res (req, res) { - return res.sendStatus(500) + postInstanceResults: { + res (req, res) { + return res.sendStatus(500) + }, }, - } + }) - setup(routes) + setupStubbedServer(routes) it('does not proceed and exits with error when parallelizing and updating instance', function () { process.env.DISABLE_API_RETRIES = 'true' @@ -912,29 +865,27 @@ describe('e2e record', () => { expect(urls).to.deep.eq([ 'POST /runs', `POST /runs/${runId}/instances`, - 'PUT /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6', + 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/tests', + 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/results', ]) }) }) }) describe('create run 422', () => { - const routes = [{ - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - res (req, res) { - return res.status(422).json({ - code: 'RUN_GROUP_NAME_NOT_UNIQUE', - message: 'Run group name cannot be used again without passing the parallel flag.', - payload: { - runUrl: 'https://dashboard.cypress.io/runs/12345', - }, - }) + setupStubbedServer(createRoutes({ + postRun: { + res (req, res) { + return res.status(422).json({ + code: 'RUN_GROUP_NAME_NOT_UNIQUE', + message: 'Run group name cannot be used again without passing the parallel flag.', + payload: { + runUrl: 'https://dashboard.cypress.io/runs/12345', + }, + }) + }, }, - }] - - setup(routes) + })) // the other 422 tests for this are in integration/cypress_spec it('errors and exits when group name is in use', function () { @@ -959,19 +910,16 @@ describe('e2e record', () => { }) describe('create run unknown 422', () => { - const routes = [{ - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - res (req, res) { - return res.status(422).json({ - code: 'SOMETHING_UNKNOWN', - message: 'An unknown message here from the server.', - }) + setupStubbedServer(createRoutes({ + postRun: { + res (req, res) { + return res.status(422).json({ + code: 'SOMETHING_UNKNOWN', + message: 'An unknown message here from the server.', + }) + }, }, - }] - - setup(routes) + })) it('errors and exits when there is an unknown 422 response', function () { return e2e.exec(this, { @@ -996,21 +944,20 @@ describe('e2e record', () => { }) describe('create run 402 - free plan exceeds monthly private tests', () => { - setup([{ - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - res (req, res) { - return res.status(402).json({ - code: 'FREE_PLAN_EXCEEDS_MONTHLY_PRIVATE_TESTS', - payload: { - used: 600, - limit: 500, - orgId: 'org-id-1234', - }, - }) + setupStubbedServer(createRoutes({ + postRun: { + res (req, res) { + return res.status(402).json({ + code: 'FREE_PLAN_EXCEEDS_MONTHLY_PRIVATE_TESTS', + payload: { + used: 600, + limit: 500, + orgId: 'org-id-1234', + }, + }) + }, }, - }]) + })) it('errors and exits when on free plan and over recorded runs limit', function () { return e2e.exec(this, { @@ -1024,21 +971,20 @@ describe('e2e record', () => { }) describe('create run 402 - free plan exceeds monthly tests', () => { - setup([{ - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - res (req, res) { - return res.status(402).json({ - code: 'FREE_PLAN_EXCEEDS_MONTHLY_TESTS', - payload: { - used: 600, - limit: 500, - orgId: 'org-id-1234', - }, - }) + setupStubbedServer(createRoutes({ + postRun: { + res (req, res) { + return res.status(402).json({ + code: 'FREE_PLAN_EXCEEDS_MONTHLY_TESTS', + payload: { + used: 600, + limit: 500, + orgId: 'org-id-1234', + }, + }) + }, }, - }]) + })) it('errors and exits when on free plan and over recorded tests limit', function () { return e2e.exec(this, { @@ -1052,19 +998,17 @@ describe('e2e record', () => { }) describe('create run 402 - parallel feature not available in plan', () => { - setup([{ - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - res (req, res) { - return res.status(402).json({ - code: 'PARALLEL_FEATURE_NOT_AVAILABLE_IN_PLAN', - payload: { - orgId: 'org-id-1234', - }, - }) - }, - }]) + setupStubbedServer(createRoutes({ + postRun: { + res (req, res) { + return res.status(402).json({ + code: 'PARALLEL_FEATURE_NOT_AVAILABLE_IN_PLAN', + payload: { + orgId: 'org-id-1234', + }, + }) + }, + } })) it('errors and exits when attempting parallel run when not available in plan', function () { return e2e.exec(this, { @@ -1078,10 +1022,7 @@ describe('e2e record', () => { }) describe('create run 402 - grouping feature not available in plan', () => { - setup([{ - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', + setupStubbedServer(createRoutes({ postRun: { res (req, res) { return res.status(402).json({ code: 'RUN_GROUPING_FEATURE_NOT_AVAILABLE_IN_PLAN', @@ -1090,7 +1031,7 @@ describe('e2e record', () => { }, }) }, - }]) + } })) it('errors and exits when attempting parallel run when not available in plan', function () { return e2e.exec(this, { @@ -1104,16 +1045,13 @@ describe('e2e record', () => { }) describe('create run 402 - unknown error', () => { - setup([{ - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', + setupStubbedServer(createRoutes({ postRun: { res (req, res) { return res.status(402).json({ error: 'Something went wrong', }) }, - }]) + } })) it('errors and exits when there\'s an unknown 402 error', function () { return e2e.exec(this, { @@ -1127,34 +1065,23 @@ describe('e2e record', () => { }) describe('create instance', () => { - const routes = [ - { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - resSchema: 'postRunResponse@2.2.0', - res: postRunResponse, - }, { - method: 'post', - url: '/runs/:id/instances', - req: 'postRunInstanceRequest@2.1.0', - resSchema: 'postRunInstanceResponse@2.1.0', + setupStubbedServer(createRoutes({ + postRunInstance: { res (req, res) { return res.sendStatus(500) }, }, - ] - - setup(routes) + })) - it('does not update instance', function () { + it('errors and exits on createInstance error', function () { process.env.DISABLE_API_RETRIES = 'true' return e2e.exec(this, { key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', - spec: 'record_pass*', + spec: '*_record_*', record: true, snapshot: true, + expectedExitCode: 1, }) .then(() => { const urls = getRequestUrls() @@ -1167,33 +1094,95 @@ describe('e2e record', () => { }) }) - describe('update instance', () => { - const routes = [ - { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - resSchema: 'postRunResponse@2.2.0', - res: postRunResponse, - }, { - method: 'post', - url: '/runs/:id/instances', - req: 'postRunInstanceRequest@2.1.0', - resSchema: 'postRunInstanceResponse@2.1.0', - res: postRunInstanceResponse, - }, { - method: 'put', - url: '/instances/:id', - req: 'putInstanceRequest@3.0.0', + describe('postInstanceTests', () => { + setupStubbedServer(createRoutes({ + postInstanceTests: { + res (req, res) { + res.sendStatus(500) + }, + }, + })) + + // it('without parallelization continues, does not post instance results', async function () { + // process.env.DISABLE_API_RETRIES = 'true' + + // return e2e.exec(this, { + // key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + // spec: '*_record.spec*', + // record: true, + // snapshot: true, + // }) + // .then(() => { + // const urls = getRequestUrls() + + // expect(urls).to.deep.eq([ + // 'POST /runs', + // 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', + // 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/tests', + // 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', + // 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/tests', + // ]) + // }) + // }) + + it('without parallelization errors and exits', async function () { + process.env.DISABLE_API_RETRIES = 'true' + + return e2e.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + spec: '*_record.spec*', + group: 'foo', + ciBuildId: 1, + record: true, + snapshot: true, + }) + .then(() => { + const urls = getRequestUrls() + + expect(urls).to.deep.eq([ + 'POST /runs', + 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', + 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/tests', + ]) + }) + }) + + it('with parallelization errors and exits', async function () { + process.env.DISABLE_API_RETRIES = 'true' + + await e2e.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + spec: '*_record.spec.js', + record: true, + group: 'foo', + ciBuildId: 'ciBuildId123', + parallel: true, + snapshot: true, + }) + .then(() => { + const urls = getRequestUrls() + + expect(urls).to.deep.eq([ + 'POST /runs', + 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', + 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/tests', + ]) + }) + }) + }) + + describe('postInstanceResults', () => { + const routes = createRoutes({ + postInstanceResults: { res (req, res) { return res.sendStatus(500) }, }, - ] + }) - setup(routes) + setupStubbedServer(routes) - it('does not update instance stdout', function () { + it('errors and exits in serial', function () { process.env.DISABLE_API_RETRIES = 'true' return e2e.exec(this, { @@ -1201,6 +1190,7 @@ describe('e2e record', () => { spec: 'record_pass*', record: true, snapshot: true, + expectedExitCode: 1, }) .then(() => { const urls = getRequestUrls() @@ -1208,59 +1198,23 @@ describe('e2e record', () => { expect(urls).to.deep.eq([ 'POST /runs', `POST /runs/${runId}/instances`, - `PUT /instances/${instanceId}`, + `POST /instances/${instanceId}/tests`, + `POST /instances/${instanceId}/results`, ]) }) }) }) describe('update instance stdout', () => { - const routes = [ - { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - resSchema: 'postRunResponse@2.2.0', - res: postRunResponse, - }, { - method: 'post', - url: '/runs/:id/instances', - req: 'postRunInstanceRequest@2.1.0', - resSchema: 'postRunInstanceResponse@2.1.0', - res: postRunInstanceResponse, - }, { - method: 'put', - url: '/instances/:id', - req: 'putInstanceRequest@3.0.0', - resSchema: 'putInstanceResponse@2.0.0', - res: sendUploadUrls, - }, { - method: 'put', - url: '/instances/:id/stdout', - req: 'putInstanceStdoutRequest@1.0.0', - resSchema: 'putInstanceStdoutRequest@1.0.0', + const routes = createRoutes({ + putInstanceStdout: { res (req, res) { return res.sendStatus(500) }, - }, { - method: 'put', - url: '/videos/:name', - res (req, res) { - return Promise.delay(500) - .then(() => { - return res.sendStatus(200) - }) - }, - }, { - method: 'put', - url: '/screenshots/:name', - res (req, res) { - return res.sendStatus(200) - }, }, - ] + }) - setup(routes) + setupStubbedServer(routes) it('warns but proceeds', function () { process.env.DISABLE_API_RETRIES = 'true' @@ -1277,60 +1231,35 @@ describe('e2e record', () => { expect(urls).to.deep.eq([ 'POST /runs', `POST /runs/${runId}/instances`, - `PUT /instances/${instanceId}`, + `POST /instances/${instanceId}/tests`, + `POST /instances/${instanceId}/results`, 'PUT /screenshots/1.png', `PUT /instances/${instanceId}/stdout`, + `POST /runs/${runId}/instances`, ]) }) }) }) describe('uploading assets', () => { - const routes = [ - { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - resSchema: 'postRunResponse@2.2.0', - res: postRunResponse, - }, { - method: 'post', - url: '/runs/:id/instances', - req: 'postRunInstanceRequest@2.1.0', - resSchema: 'postRunInstanceResponse@2.1.0', - res: postRunInstanceResponse, - }, { - method: 'put', - url: '/instances/:id', - req: 'putInstanceRequest@3.0.0', - resSchema: 'putInstanceResponse@2.0.0', - res: sendUploadUrls, - }, { - method: 'put', - url: '/instances/:id/stdout', - req: 'putInstanceStdoutRequest@1.0.0', - res (req, res) { - return res.sendStatus(200) - }, - }, { - method: 'put', - url: '/videos/:name', + const routes = createRoutes({ + + putVideo: { res (req, res) { return Promise.delay(500) .then(() => { return res.sendStatus(500) }) }, - }, { - method: 'put', - url: '/screenshots/:name', + }, + putScreenshots: { res (req, res) { return res.sendStatus(500) }, }, - ] + }) - setup(routes, { + setupStubbedServer(routes, { videoUploadOnPasses: true, }) @@ -1347,7 +1276,8 @@ describe('e2e record', () => { expect(urls).to.have.members([ 'POST /runs', `POST /runs/${runId}/instances`, - `PUT /instances/${instanceId}`, + `POST /instances/${instanceId}/tests`, + `POST /instances/${instanceId}/results`, 'PUT /videos/video.mp4', 'PUT /screenshots/1.png', `PUT /instances/${instanceId}/stdout`, @@ -1359,56 +1289,49 @@ describe('e2e record', () => { describe('api retries on error', () => { let count = 0 - const routes = defaultRoutes.slice(0) - - routes[0] = { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - res (req, res) { - count += 1 + const routes = createRoutes({ + postRun: { + res (req, res) { + count += 1 - if (count === 4) { - return res.json(postRunResponse) - } + if (count === 4) { + return res.json(postRunResponse) + } - return res.sendStatus(500) + return res.sendStatus(500) + }, }, - } - - routes[1] = { - method: 'post', - url: '/runs/:id/instances', - req: 'postRunInstanceRequest@2.1.0', - resSchema: 'postRunInstanceResponse@2.1.0', - res (req, res) { - count += 1 + postRunInstance: { + res (req, res) { + count += 1 - if (count === 5) { - return res.sendStatus(500) - } + if (count === 5) { + return res.sendStatus(500) + } + + if (count === 6) { + return res.json({ + instanceId, + spec: 'cypress/integration/record_pass_spec.js', + estimatedWallClockDuration: 5000, + totalInstances: 1, + claimedInstances: 1, + }) + } - if (count === 6) { return res.json({ instanceId, - spec: 'cypress/integration/record_pass_spec.js', - estimatedWallClockDuration: 5000, - totalInstances: 1, - claimedInstances: 1, + spec: null, + estimatedWallClockDuration: null, + totalInstances: 0, + claimedInstances: 0, }) - } - - return res.json({ - instanceId, - spec: null, - estimatedWallClockDuration: null, - totalInstances: 0, - claimedInstances: 0, - }) + }, }, - } - setup(routes) + }) + + setupStubbedServer(routes) it('warns and does not create or update instances', function () { process.env.API_RETRY_INTERVALS = '1000,2000,3000' @@ -1433,7 +1356,8 @@ describe('e2e record', () => { 'POST /runs', 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', - 'PUT /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6', + 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/tests', + 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/results', 'PUT /screenshots/1.png', 'PUT /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/stdout', 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', @@ -1446,33 +1370,30 @@ describe('e2e record', () => { describe('api interaction warnings', () => { describe('create run warnings', () => { describe('grace period - over private tests limit', () => { - const routes = defaultRoutes.slice() - - routes[0] = { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - resSchema: 'postRunResponse@2.2.0', - res (req, res) { - return res.status(200).json({ - runId, - groupId, - machineId, - runUrl, - tags, - warnings: [{ - name: 'foo', - message: 'foo', - code: 'FREE_PLAN_IN_GRACE_PERIOD_EXCEEDS_MONTHLY_PRIVATE_TESTS', - limit: 500, - gracePeriodEnds: '2999-12-31', - orgId: 'org-id-1234', - }], - }) + const mockServer = setupStubbedServer(createRoutes({ + postRun: { + res (req, res) { + mockServer.setSpecs(req) + + return res.status(200).json({ + runId, + groupId, + machineId, + runUrl, + tags, + warnings: [{ + name: 'foo', + message: 'foo', + code: 'FREE_PLAN_IN_GRACE_PERIOD_EXCEEDS_MONTHLY_PRIVATE_TESTS', + limit: 500, + gracePeriodEnds: '2999-12-31', + orgId: 'org-id-1234', + }], + }) + }, }, - } - setup(routes) + })) it('warns when over private test recordings', function () { return e2e.exec(this, { @@ -1485,33 +1406,29 @@ describe('e2e record', () => { }) describe('grace period - over tests limit', () => { - const routes = defaultRoutes.slice() - - routes[0] = { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - resSchema: 'postRunResponse@2.2.0', - res (req, res) { - return res.status(200).json({ - runId, - groupId, - machineId, - runUrl, - tags, - warnings: [{ - name: 'foo', - message: 'foo', - code: 'FREE_PLAN_IN_GRACE_PERIOD_EXCEEDS_MONTHLY_TESTS', - limit: 500, - gracePeriodEnds: '2999-12-31', - orgId: 'org-id-1234', - }], - }) + const mockServer = setupStubbedServer(createRoutes({ + postRun: { + res (req, res) { + mockServer.setSpecs(req) + + return res.status(200).json({ + runId, + groupId, + machineId, + runUrl, + tags, + warnings: [{ + name: 'foo', + message: 'foo', + code: 'FREE_PLAN_IN_GRACE_PERIOD_EXCEEDS_MONTHLY_TESTS', + limit: 500, + gracePeriodEnds: '2999-12-31', + orgId: 'org-id-1234', + }], + }) + }, }, - } - - setup(routes) + })) it('warns when over test recordings', function () { return e2e.exec(this, { @@ -1524,32 +1441,28 @@ describe('e2e record', () => { }) describe('grace period - parallel feature', () => { - const routes = defaultRoutes.slice() - - routes[0] = { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - resSchema: 'postRunResponse@2.2.0', - res (req, res) { - return res.status(200).json({ - runId, - groupId, - machineId, - runUrl, - tags, - warnings: [{ - name: 'foo', - message: 'foo', - code: 'FREE_PLAN_IN_GRACE_PERIOD_PARALLEL_FEATURE', - gracePeriodEnds: '2999-12-31', - orgId: 'org-id-1234', - }], - }) + const mockServer = setupStubbedServer(createRoutes({ + postRun: { + res (req, res) { + mockServer.setSpecs(req) + + return res.status(200).json({ + runId, + groupId, + machineId, + runUrl, + tags, + warnings: [{ + name: 'foo', + message: 'foo', + code: 'FREE_PLAN_IN_GRACE_PERIOD_PARALLEL_FEATURE', + gracePeriodEnds: '2999-12-31', + orgId: 'org-id-1234', + }], + }) + }, }, - } - - setup(routes) + })) it('warns when using parallel feature', function () { return e2e.exec(this, { @@ -1562,32 +1475,28 @@ describe('e2e record', () => { }) describe('grace period - grouping feature', () => { - const routes = defaultRoutes.slice() - - routes[0] = { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - resSchema: 'postRunResponse@2.2.0', - res (req, res) { - return res.status(200).json({ - runId, - groupId, - machineId, - runUrl, - tags, - warnings: [{ - name: 'foo', - message: 'foo', - code: 'PLAN_IN_GRACE_PERIOD_RUN_GROUPING_FEATURE_USED', - gracePeriodEnds: '2999-12-31', - orgId: 'org-id-1234', - }], - }) + const mockServer = setupStubbedServer(createRoutes({ + postRun: { + res (req, res) { + mockServer.setSpecs(req) + + return res.status(200).json({ + runId, + groupId, + machineId, + runUrl, + tags, + warnings: [{ + name: 'foo', + message: 'foo', + code: 'PLAN_IN_GRACE_PERIOD_RUN_GROUPING_FEATURE_USED', + gracePeriodEnds: '2999-12-31', + orgId: 'org-id-1234', + }], + }) + }, }, - } - - setup(routes) + })) it('warns when using parallel feature', function () { return e2e.exec(this, { @@ -1600,33 +1509,29 @@ describe('e2e record', () => { }) describe('paid plan - over private tests limit', () => { - const routes = defaultRoutes.slice() - - routes[0] = { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - resSchema: 'postRunResponse@2.2.0', - res (req, res) { - return res.status(200).json({ - runId, - groupId, - machineId, - runUrl, - tags, - warnings: [{ - name: 'foo', - message: 'foo', - code: 'PAID_PLAN_EXCEEDS_MONTHLY_PRIVATE_TESTS', - used: 700, - limit: 500, - orgId: 'org-id-1234', - }], - }) + const mockServer = setupStubbedServer(createRoutes({ + postRun: { + res (req, res) { + mockServer.setSpecs(req) + + return res.status(200).json({ + runId, + groupId, + machineId, + runUrl, + tags, + warnings: [{ + name: 'foo', + message: 'foo', + code: 'PAID_PLAN_EXCEEDS_MONTHLY_PRIVATE_TESTS', + used: 700, + limit: 500, + orgId: 'org-id-1234', + }], + }) + }, }, - } - - setup(routes) + })) it('warns when over private test recordings', function () { return e2e.exec(this, { @@ -1639,33 +1544,29 @@ describe('e2e record', () => { }) describe('paid plan - over tests limit', () => { - const routes = defaultRoutes.slice() - - routes[0] = { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - resSchema: 'postRunResponse@2.2.0', - res (req, res) { - return res.status(200).json({ - runId, - groupId, - machineId, - runUrl, - tags, - warnings: [{ - name: 'foo', - message: 'foo', - code: 'PAID_PLAN_EXCEEDS_MONTHLY_TESTS', - used: 700, - limit: 500, - orgId: 'org-id-1234', - }], - }) + const mockServer = setupStubbedServer(createRoutes({ + postRun: { + res (req, res) { + mockServer.setSpecs(req) + + return res.status(200).json({ + runId, + groupId, + machineId, + runUrl, + tags, + warnings: [{ + name: 'foo', + message: 'foo', + code: 'PAID_PLAN_EXCEEDS_MONTHLY_TESTS', + used: 700, + limit: 500, + orgId: 'org-id-1234', + }], + }) + }, }, - } - - setup(routes) + })) it('warns when over test recordings', function () { return e2e.exec(this, { @@ -1678,33 +1579,29 @@ describe('e2e record', () => { }) describe('free plan - over tests limit v2', () => { - const routes = defaultRoutes.slice() - - routes[0] = { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - resSchema: 'postRunResponse@2.2.0', - res (req, res) { - return res.status(200).json({ - runId, - groupId, - machineId, - runUrl, - tags, - warnings: [{ - name: 'FreePlanExceedsMonthlyTests', - message: 'Warning from Cypress Dashboard: Organization with free plan has exceeded monthly test recordings limit.', - code: 'FREE_PLAN_EXCEEDS_MONTHLY_TESTS_V2', - used: 700, - limit: 500, - orgId: 'org-id-1234', - }], - }) + const mockServer = setupStubbedServer(createRoutes({ + postRun: { + res (req, res) { + mockServer.setSpecs(req) + + return res.status(200).json({ + runId, + groupId, + machineId, + runUrl, + tags, + warnings: [{ + name: 'FreePlanExceedsMonthlyTests', + message: 'Warning from Cypress Dashboard: Organization with free plan has exceeded monthly test recordings limit.', + code: 'FREE_PLAN_EXCEEDS_MONTHLY_TESTS_V2', + used: 700, + limit: 500, + orgId: 'org-id-1234', + }], + }) + }, }, - } - - setup(routes) + })) it('warns when over test recordings', function () { return e2e.exec(this, { @@ -1717,17 +1614,14 @@ describe('e2e record', () => { }) describe('unknown warning', () => { - const routes = defaultRoutes.slice() - - routes[0] = { - method: 'post', - url: '/runs', - req: 'postRunRequest@2.2.0', - resSchema: 'postRunResponse@2.2.0', - res: postRunResponseWithWarnings, - } - - setup(routes) + const mockServer = setupStubbedServer(createRoutes({ + postRun: { + res: (req, res) => { + mockServer.setSpecs(req) + res.json(postRunResponseWithWarnings) + }, + }, + })) it('warns with unknown warning code', function () { return e2e.exec(this, { diff --git a/packages/server/test/support/fixtures/projects/e2e/cypress/integration/a_record.spec.js b/packages/server/test/support/fixtures/projects/e2e/cypress/integration/a_record.spec.js new file mode 100644 index 000000000000..e47a57dd4dce --- /dev/null +++ b/packages/server/test/support/fixtures/projects/e2e/cypress/integration/a_record.spec.js @@ -0,0 +1,5 @@ +describe('a spec', () => { + it('a test', () => { + + }) +}) diff --git a/packages/server/test/support/fixtures/projects/e2e/cypress/integration/a_record_instantfail.spec.js b/packages/server/test/support/fixtures/projects/e2e/cypress/integration/a_record_instantfail.spec.js new file mode 100644 index 000000000000..8f9316a6477b --- /dev/null +++ b/packages/server/test/support/fixtures/projects/e2e/cypress/integration/a_record_instantfail.spec.js @@ -0,0 +1,14 @@ +Cypress.run = () => { + // force a plugin crash immediately + // e.g. to test when spec is skipped that Cypress.run is never called + Cypress.backend('task', { + task: 'plugins:crash', + arg: '', + timeout: 6000, + }) +} + +describe('a spec', () => { + it('a test', () => { + }) +}) diff --git a/packages/server/test/support/fixtures/projects/e2e/cypress/integration/b_record.spec.js b/packages/server/test/support/fixtures/projects/e2e/cypress/integration/b_record.spec.js new file mode 100644 index 000000000000..f0a45633afa2 --- /dev/null +++ b/packages/server/test/support/fixtures/projects/e2e/cypress/integration/b_record.spec.js @@ -0,0 +1,5 @@ +describe('b spec', () => { + it('b test', () => { + + }) +}) diff --git a/packages/server/test/support/fixtures/projects/e2e/cypress/integration/config_record_spec.js b/packages/server/test/support/fixtures/projects/e2e/cypress/integration/config_record_spec.js new file mode 100644 index 000000000000..03c66ec02552 --- /dev/null +++ b/packages/server/test/support/fixtures/projects/e2e/cypress/integration/config_record_spec.js @@ -0,0 +1,20 @@ +Cypress.config('defaultCommandTimeout', 1111) +Cypress.config('pageLoadTimeout', 2222) +describe('record pass', { defaultCommandTimeout: 1234 }, () => { + Cypress.config('pageLoadTimeout', 3333) + it('passes', { env: { foo: true }, retries: 2 }, () => { + Cypress.config('defaultCommandTimeout', 4444) + cy.visit('/scrollable.html') + cy.viewport(400, 400) + cy.get('#box') + cy.screenshot('yay it passes') + }) + + it('is pending') + + // eslint-disable-next-line + it.skip('is pending due to .skip', () => {}) + it('is skipped due to browser', { browser: 'edge' }, () => {}) +}) + +// add retries and test in snapshot / assertion diff --git a/packages/server/test/support/fixtures/projects/e2e/cypress/integration/record_pass_spec.js b/packages/server/test/support/fixtures/projects/e2e/cypress/integration/record_pass_spec.js index be912394ee0b..db11108c95b5 100644 --- a/packages/server/test/support/fixtures/projects/e2e/cypress/integration/record_pass_spec.js +++ b/packages/server/test/support/fixtures/projects/e2e/cypress/integration/record_pass_spec.js @@ -1,8 +1,7 @@ /* eslint-disable no-undef */ describe('record pass', () => { - it('passes', () => { + it('passes', { env: { foo: true } }, () => { cy.visit('/scrollable.html') - cy.viewport(400, 400) cy.get('#box') cy.screenshot('yay it passes') diff --git a/packages/server/test/support/fixtures/projects/e2e/cypress/plugins/index.js b/packages/server/test/support/fixtures/projects/e2e/cypress/plugins/index.js index b43101034a84..f1df5879287b 100644 --- a/packages/server/test/support/fixtures/projects/e2e/cypress/plugins/index.js +++ b/packages/server/test/support/fixtures/projects/e2e/cypress/plugins/index.js @@ -82,6 +82,11 @@ module.exports = (on, config, mode) => { throw new Error(message) }, + 'plugins:crash' (message) { + console.log('\nPURPOSEFULLY CRASHING THE PLUGIN PROCESS FROM TEST') + process.exit(1) + }, + 'ensure:pixel:color' ({ name, colors, devicePixelRatio }) { const imagePath = path.join(__dirname, '..', 'screenshots', `${name}.png`) diff --git a/packages/server/test/support/helpers/resultsUtils.ts b/packages/server/test/support/helpers/resultsUtils.ts index b8853a6c1384..7863f791a959 100644 --- a/packages/server/test/support/helpers/resultsUtils.ts +++ b/packages/server/test/support/helpers/resultsUtils.ts @@ -74,8 +74,7 @@ const normalizeTestTimings = function (obj, timings) { export const expectRunsToHaveCorrectTimings = (runs = []) => { runs.forEach((run) => { - expect(run.cypressConfig).to.be.a('object') - run.cypressConfig = {} + expect(run.config).to.not.exist expectStartToBeBeforeEnd(run, 'stats.wallClockStartedAt', 'stats.wallClockEndedAt') expectStartToBeBeforeEnd(run, 'reporterStats.start', 'reporterStats.end') @@ -163,7 +162,7 @@ export const expectRunsToHaveCorrectTimings = (runs = []) => { } }) } catch (e) { - e.message = `Error during validation for test "${test.title.join(' / ')}"\n${e.message}` + e.message = `Error during validation for test \n${e.message}` throw e } }) diff --git a/packages/server/test/support/helpers/serverStub.ts b/packages/server/test/support/helpers/serverStub.ts new file mode 100644 index 000000000000..028e638f85e9 --- /dev/null +++ b/packages/server/test/support/helpers/serverStub.ts @@ -0,0 +1,298 @@ +import _ from 'lodash' +import Bluebird from 'bluebird' +import bodyParser from 'body-parser' +import { api as jsonSchemas } from '@cypress/json-schemas' +import e2e from './e2e' + +export const postRunResponseWithWarnings = jsonSchemas.getExample('postRunResponse')('2.2.0') + +export const postRunInstanceResponse = jsonSchemas.getExample('postRunInstanceResponse')('2.1.0') + +export const postInstanceTestsResponse = jsonSchemas.getExample('postInstanceTestsResponse')('1.0.0') + +postInstanceTestsResponse.actions = [] +export const postRunResponse = _.assign({}, postRunResponseWithWarnings, { warnings: [] }) + +type DeepPartial = { + [P in keyof T]?: DeepPartial; +}; +const sendUploadUrls = function (req, res) { + const { body } = req + + let num = 0 + + const json = {} as any + + if (body.video) { + json.videoUploadUrl = 'http://localhost:1234/videos/video.mp4' + } + + const screenshotUploadUrls = _.map(body.screenshots, (s) => { + num += 1 + + return { + screenshotId: s.screenshotId, + uploadUrl: `http://localhost:1234/screenshots/${num}.png`, + } + }) + + json.screenshotUploadUrls = screenshotUploadUrls + + return res.json(json) +} +const mockServerState = { + requests: [], + setSpecs (req) { + mockServerState.specs = [...req.body.specs] + mockServerState.allSpecs = [...req.body.specs] + }, + allSpecs: [], + specs: [], +} + +const routeHandlers = { + postRun: { + method: 'post', + url: '/runs', + req: 'postRunRequest@2.3.0', + resSchema: 'postRunResponse@2.2.0', + res: (req, res) => { + if (!req.body.specs) { + throw new Error('expected for Test Runner to post specs') + } + + mockServerState.setSpecs(req) + + return res.json(postRunResponse) + }, + }, + postRunInstance: { + method: 'post', + url: '/runs/:id/instances', + req: 'postRunInstanceRequest@2.1.0', + resSchema: 'postRunInstanceResponse@2.1.0', + res: (req, res) => { + console.log(mockServerState.allSpecs.length, mockServerState.specs.length) + const response = { + ...postRunInstanceResponse, + spec: mockServerState.specs.shift() || null, + claimedInstances: mockServerState.allSpecs.length - mockServerState.specs.length, + totalInstances: mockServerState.allSpecs.length, + } + + console.log('response', response) + + return res.json(response) + }, + }, + postInstanceTests: { + method: 'post', + url: '/instances/:id/tests', + req: 'postInstanceTestsRequest@1.0.0', + resSchema: 'postInstanceTestsResponse@1.0.0', + res: postInstanceTestsResponse, + }, + postInstanceResults: { + method: 'post', + url: '/instances/:id/results', + req: 'postInstanceResultsRequest@1.0.0', + resSchema: 'postInstanceResultsResponse@1.0.0', + res: sendUploadUrls, + }, + putInstanceStdout: { + method: 'put', + url: '/instances/:id/stdout', + req: 'putInstanceStdoutRequest@1.0.0', + res (req, res) { + return res.sendStatus(200) + }, + }, + putVideo: { + method: 'put', + url: '/videos/:name', + res (req, res) { + return Bluebird.delay(200) + .then(() => { + return res.sendStatus(200) + }) + }, + }, + putScreenshots: { + method: 'put', + url: '/screenshots/:name', + res (req, res) { + return res.sendStatus(200) + }, + }, + +} + +export const createRoutes = (props: DeepPartial) => { + return _.values(_.merge(_.cloneDeep(routeHandlers), props)) +} + +beforeEach(() => { + mockServerState.requests.length = 0 + mockServerState.specs.length = 0 + mockServerState.allSpecs.length = 0 +}) + +export const getRequestUrls = () => { + return _.map(mockServerState.requests, 'url') +} + +export const getRequests = () => { + return mockServerState.requests +} + +const getSchemaErr = (tag, err, schema) => { + return { + errors: err.errors, + object: err.object, + example: err.example, + message: `${tag} should follow ${schema} schema`, + } +} + +const getResponse = function (responseSchema) { + if (!responseSchema) { + throw new Error('No response schema supplied') + } + + if (_.isObject(responseSchema)) { + return responseSchema + } + + const [name, version] = responseSchema.split('@') + + return jsonSchemas.getExample(name)(version) +} + +const sendResponse = function (req, res, responseBody) { + return new Promise((resolve) => { + const _writeRaw = res._writeRaw + + res._writeRaw = function () { + resolve() + + return _writeRaw.apply(this, arguments) + } + + if (_.isFunction(responseBody)) { + return responseBody(req, res) + } + + res.json(getResponse(responseBody)) + resolve() + }) +} + +const ensureSchema = function (expectedRequestSchema, responseBody, expectedResponseSchema) { + let reqName; let reqVersion + + if (expectedRequestSchema) { + [reqName, reqVersion] = expectedRequestSchema.split('@') + } + + return async function (req, res) { + const { body } = req + + try { + if (expectedRequestSchema) { + jsonSchemas.assertSchema(reqName, reqVersion)(body) + } + + res.expectedResponseSchema = expectedResponseSchema + + await sendResponse(req, res, responseBody) + + const key = [req.method, req.url].join(' ') + + mockServerState.requests.push({ + url: key, + body, + }) + } catch (err) { + console.log('Schema Error:', err.message) + + return res.status(412).json(getSchemaErr('request', err, expectedRequestSchema)) + } + } +} + +const assertResponseBodySchema = function (req, res, next) { + const oldWrite = res.write + const oldEnd = res.end + + const chunks = [] + + res.write = (chunk) => { + // buffer the response, we'll really write it on end + return chunks.push(chunk) + } + + res.end = function (chunk) { + if (chunk) { + chunks.push(chunk) + } + + res.write = oldWrite + res.end = oldEnd + + if (res.expectedResponseSchema && _.inRange(res.statusCode, 200, 299)) { + const body = JSON.parse(Buffer.concat(chunks).toString('utf8')) + + const [resName, resVersion] = res.expectedResponseSchema.split('@') + + try { + jsonSchemas.assertSchema(resName, resVersion)(body) + } catch (err) { + console.log('Schema Error:', err.message) + + return res.status(412).json(getSchemaErr('response', err, res.expectedResponseSchema)) + } + } + + chunks.map((chunk) => { + return res.write(chunk) + }) + + return res.end() + } + + return next() +} + +const onServer = (routes) => { + return (function (app) { + app.use(bodyParser.json()) + + app.use(assertResponseBodySchema) + + return _.each(routes, (route) => { + return app[route.method](route.url, ensureSchema( + route.req, + route.res, + route.resSchema, + )) + }) + }) +} + +export const setupStubbedServer = (routes, settings = {}) => { + e2e.setup({ + settings: _.extend({ + projectId: 'pid123', + videoUploadOnPasses: false, + }, settings), + servers: [{ + port: 1234, + onServer: onServer(routes), + }, { + port: 3131, + static: true, + }], + }) + + return mockServerState +} diff --git a/packages/server/test/unit/api_spec.js b/packages/server/test/unit/api_spec.js index 316964710c27..6d365e03010a 100644 --- a/packages/server/test/unit/api_spec.js +++ b/packages/server/test/unit/api_spec.js @@ -344,6 +344,10 @@ describe('lib/api', () => { remoteOrigin: 'https://github.com/foo/bar.git', }, specs: ['foo.js', 'bar.js'], + runnerCapabilities: { + 'dynamicSpecsInSerialMode': true, + 'skipSpecAction': true, + }, } }) @@ -559,38 +563,36 @@ describe('lib/api', () => { error: 'err msg', video: true, screenshots: [], - cypressConfig: {}, reporterStats: {}, - stdout: null, } - this.putProps = _.omit(this.updateProps, 'instanceId') + this.postProps = _.pick(this.updateProps, 'stats', 'video', 'screenshots', 'reporterStats') }) - it('PUTs /instances/:id', function () { + it('POSTs /instances/:id/results', function () { nock(API_BASEURL) - .matchHeader('x-route-version', '3') + .matchHeader('x-route-version', '1') .matchHeader('x-os-name', 'linux') .matchHeader('x-cypress-version', pkg.version) - .put('/instances/instance-id-123', this.putProps) + .post('/instances/instance-id-123/results', this.postProps) .reply(200) - return api.updateInstance(this.updateProps) + return api.postInstanceResults(this.updateProps) }) it('PUT /instances/:id failure formatting', () => { nock(API_BASEURL) - .matchHeader('x-route-version', '3') + .matchHeader('x-route-version', '1') .matchHeader('x-os-name', 'linux') .matchHeader('x-cypress-version', pkg.version) - .put('/instances/instance-id-123') + .post('/instances/instance-id-123/results') .reply(422, { errors: { tests: ['is required'], }, }) - return api.updateInstance({ instanceId: 'instance-id-123' }) + return api.postInstanceResults({ instanceId: 'instance-id-123' }) .then(() => { throw new Error('should have thrown here') }).catch((err) => { @@ -610,14 +612,14 @@ describe('lib/api', () => { it('handles timeouts', () => { nock(API_BASEURL) - .matchHeader('x-route-version', '3') + .matchHeader('x-route-version', '1') .matchHeader('x-os-name', 'linux') .matchHeader('x-cypress-version', pkg.version) - .put('/instances/instance-id-123') + .post('/instances/instance-id-123/results') .socketDelay(5000) .reply(200, {}) - return api.updateInstance({ + return api.postInstanceResults({ instanceId: 'instance-id-123', timeout: 100, }) @@ -629,23 +631,23 @@ describe('lib/api', () => { }) it('sets timeout to 60 seconds', () => { - sinon.stub(api.rp, 'put').resolves() + sinon.stub(api.rp, 'post').resolves() - return api.updateInstance({}) + return api.postInstanceResults({}) .then(() => { - expect(api.rp.put).to.be.calledWithMatch({ timeout: 60000 }) + expect(api.rp.post).to.be.calledWithMatch({ timeout: 60000 }) }) }) it('tags errors', function () { nock(API_BASEURL) - .matchHeader('x-route-version', '2') + .matchHeader('x-route-version', '1') .matchHeader('authorization', 'Bearer auth-token-123') .matchHeader('accept-encoding', /gzip/) - .put('/instances/instance-id-123', this.putProps) + .post('/instances/instance-id-123/results', this.postProps) .reply(500, {}) - return api.updateInstance(this.updateProps) + return api.postInstanceResults(this.updateProps) .then(() => { throw new Error('should have thrown here') }).catch((err) => { diff --git a/packages/server/test/unit/modes/record_spec.js b/packages/server/test/unit/modes/record_spec.js index 9b0456592a78..47ade1c03708 100644 --- a/packages/server/test/unit/modes/record_spec.js +++ b/packages/server/test/unit/modes/record_spec.js @@ -439,27 +439,24 @@ describe('lib/modes/record', () => { }) }) - it('does not createException when statusCode is 503', () => { + it('errors when statusCode is 503', async () => { const err = new Error('foo') err.statusCode = 503 - sinon.spy(logger, 'createException') + sinon.spy(errors, 'get') sinon.stub(api, 'retryWithBackoff').rejects(err) - return recordMode.createInstance({ + await expect(recordMode.createInstance({ runId: 'run-123', groupId: 'group-123', machineId: 'machine-123', platform: {}, spec: { relative: 'cypress/integration/app_spec.coffee' }, - }) - .then((ret) => { - expect(ret).to.be.null + })).to.be.rejected - expect(logger.createException).not.to.be.called - }) + expect(errors.get).to.have.been.calledWith('DASHBOARD_CANNOT_PROCEED_IN_SERIAL') }) }) @@ -511,9 +508,40 @@ describe('lib/modes/record', () => { }) }) - context('.updateInstance', () => { + context('.postInstanceTests', () => { + beforeEach(function () { + sinon.stub(api, 'postInstanceTests') + sinon.stub(ciProvider, 'ciParams').returns({}) + sinon.stub(ciProvider, 'provider').returns('') + sinon.stub(ciProvider, 'commitDefaults').returns({}) + + this.options = { + results: {}, + captured: '', + } + }) + + it('retries with backoff strategy', function () { + sinon.stub(api, 'retryWithBackoff').yields().resolves() + + recordMode._postInstanceTests(this.options) + + expect(api.retryWithBackoff).to.be.called + }) + + it('logs on retry', function () { + sinon.stub(api, 'retryWithBackoff').yields().resolves() + + return recordMode._postInstanceTests(this.options) + .then(() => { + expect(api.retryWithBackoff).to.be.calledOnce + }) + }) + }) + + context('.postInstanceResults', () => { beforeEach(function () { - sinon.stub(api, 'updateInstance') + sinon.stub(api, 'postInstanceResults') sinon.stub(ciProvider, 'ciParams').returns({}) sinon.stub(ciProvider, 'provider').returns('') sinon.stub(ciProvider, 'commitDefaults').returns({}) @@ -527,7 +555,7 @@ describe('lib/modes/record', () => { it('retries with backoff strategy', function () { sinon.stub(api, 'retryWithBackoff').yields().resolves() - recordMode.updateInstance(this.options) + recordMode.postInstanceResults(this.options) expect(api.retryWithBackoff).to.be.called }) @@ -535,7 +563,7 @@ describe('lib/modes/record', () => { it('logs on retry', function () { sinon.stub(api, 'retryWithBackoff').yields().resolves() - return recordMode.updateInstance(this.options) + return recordMode.postInstanceResults(this.options) .then(() => { expect(api.retryWithBackoff).to.be.calledOnce }) diff --git a/packages/server/test/unit/routes_util_spec.js b/packages/server/test/unit/routes_util_spec.js index f9284848cd9e..bf55a23b8787 100644 --- a/packages/server/test/unit/routes_util_spec.js +++ b/packages/server/test/unit/routes_util_spec.js @@ -24,8 +24,12 @@ describe('lib/util/routes', () => { expect(apiRoutes.instances(123)).to.eq('http://localhost:1234/runs/123/instances') }) - it('instance', () => { - expect(apiRoutes.instance(123)).to.eq('http://localhost:1234/instances/123') + it('instanceTests', () => { + expect(apiRoutes.instanceTests(123)).to.eq('http://localhost:1234/instances/123/tests') + }) + + it('instanceResults', () => { + expect(apiRoutes.instanceResults(123)).to.eq('http://localhost:1234/instances/123/results') }) it('projects', () => { diff --git a/yarn.lock b/yarn.lock index da4e06e7846f..070d8f2dfb66 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2132,6 +2132,17 @@ lodash.merge "^4.6.2" lodash.omit "^4.5.0" +"@cypress/json-schemas@5.37.3": + version "5.37.3" + resolved "https://registry.yarnpkg.com/@cypress/json-schemas/-/json-schemas-5.37.3.tgz#2aed2fadc9533cb8d812ab27df77ea40ccf17759" + integrity sha512-vMhydN3Ysx+d/PnbOixPzlWdvaOeHmano61D2Cwshexo3F8L68xsXqF6zy3T6teNWHXTMtNZ6fQUpcy3r082XQ== + dependencies: + "@cypress/schema-tools" "4.7.7" + lodash "^4.17.21" + lodash.clonedeep "^4.5.0" + lodash.merge "^4.6.2" + lodash.omit "^4.5.0" + "@cypress/listr-verbose-renderer@^0.4.1": version "0.4.1" resolved "https://registry.yarnpkg.com/@cypress/listr-verbose-renderer/-/listr-verbose-renderer-0.4.1.tgz#a77492f4b11dcc7c446a34b3e28721afd33c642a" @@ -2228,6 +2239,24 @@ quote "0.4.0" ramda "0.25.0" +"@cypress/schema-tools@4.7.7": + version "4.7.7" + resolved "https://registry.yarnpkg.com/@cypress/schema-tools/-/schema-tools-4.7.7.tgz#251a9864caba0eded884ff5c71de16c76dbf556a" + integrity sha512-RRzksoJIXDTeUjt7YE9xAhOynqc7R+j8Tx8ebpkSPJB6Z3WujdLP0sigVh2AV24G/CySOvJGuQQY94aBEpCZaA== + dependencies: + "@bahmutov/all-paths" "1.0.2" + "@bahmutov/is-my-json-valid" "2.17.3" + "@types/ramda" "0.25.47" + debug "4.3.1" + json-stable-stringify "1.0.1" + json2md "1.6.3" + lodash.camelcase "4.3.0" + lodash.get "4.4.2" + lodash.reduce "^4.6.0" + lodash.set "4.3.2" + quote "0.4.0" + ramda "0.25.0" + "@cypress/set-commit-status@1.3.4": version "1.3.4" resolved "https://registry.yarnpkg.com/@cypress/set-commit-status/-/set-commit-status-1.3.4.tgz#9c96e6b8c192de5723a995910ccdcca60f6c17fb" @@ -22755,6 +22784,11 @@ lodash@4.17.4: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" integrity sha1-eCA6TRwyiuHYbcpkYONptX9AVa4= +lodash@^4.17.21: + version "4.17.21" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" + integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== + log-ok@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/log-ok/-/log-ok-0.1.1.tgz#bea3dd36acd0b8a7240d78736b5b97c65444a334"