feat: dashboard orchestration (#14925)

* test orchestration

* bump json-schemas

* fix unit

* fix server-ct build

* add @types/node

* add runtime to resolved config, send resolved config to postInstanceTests

* add missing fixture spec, support sending test config for skipped and proper title

* spec prior., refactor recording tests, update specs

* remove unneded utils, fix e2e_record_spec, rename testConfig, fix unit

* fix bug with spec SKIP not stopping test execution

* bump json-schemas

* bump json schemas 2

* refactor exit early and error logic, add runnerCapabilities + bump json schemas

* update yarn.lock

* fix referenceError

* fix: skipAction -> skipSpecAction, muteAction -> muteTestAction

* exit with non-zero for canceled runs, improve messaging for canceled runs

* fixup unit tests

* colorize run cancelation messages

packages/driver/cypress/integration/commands/angular_spec.js

@@ -28,7 +28,7 @@ describe('src/cy/commands/angular', () => {
  cy.state('window').angular = this.angular
  })
 
- it('throws when cannot find angular', (done) => {
+ it('throws when cannot find angular', { retries: 2 }, (done) => {
  delete cy.state('window').angular
 
  cy.on('fail', (err) => {

packages/driver/src/cy/testConfigOverrides.ts

@@ -44,28 +44,30 @@ function mutateConfiguration (testConfigOverride, config, env) {
  return restoreConfigFn
 }
 
-function getResolvedTestConfigOverride (test) {
+// this is called during test onRunnable time
+// in order to resolve the test config upfront before test runs
+export function getResolvedTestConfigOverride (test) {
  let curParent = test.parent
 
- const cfgs = [test.cfg]
+ const testConfig = [test._testConfig]
 
  while (curParent) {
- if (curParent.cfg) {
- cfgs.push(curParent.cfg)
+ if (curParent._testConfig) {
+ testConfig.push(curParent._testConfig)
  }
 
  curParent = curParent.parent
  }
 
- return _.reduceRight(cfgs, (acc, cfg) => _.extend(acc, cfg), {})
+ return _.reduceRight(testConfig, (acc, opts) => _.extend(acc, opts), {})
 }
 
 class TestConfigOverride {
  private restoreTestConfigFn: Nullable<() => void> = null
  restoreAndSetTestConfigOverrides (test, config, env) {
  if (this.restoreTestConfigFn) this.restoreTestConfigFn()
 
- const resolvedTestConfig = getResolvedTestConfigOverride(test)
+ const resolvedTestConfig = test._testConfig || {}
 
  this.restoreTestConfigFn = mutateConfiguration(resolvedTestConfig, config, env)
  }

packages/driver/src/cypress.js

@@ -134,6 +134,7 @@ class $Cypress {
  _.extend(this, browserInfo(config))
 
  this.state = $SetterGetter.create({})
+ this.originalConfig = _.cloneDeep(config)
  this.config = $SetterGetter.create(config)
  this.env = $SetterGetter.create(env)
  this.getFirefoxGcInterval = $FirefoxForcedGc.createIntervalGetter(this)

packages/driver/src/cypress/mocha.js

@@ -33,9 +33,10 @@ const suiteAfterEach = Suite.prototype.afterEach
 delete window.mocha
 delete window.Mocha
 
-function invokeFnWithOriginalTitle (ctx, originalTitle, mochaArgs, fn) {
+function invokeFnWithOriginalTitle (ctx, originalTitle, mochaArgs, fn, _testConfig) {
  const ret = fn.apply(ctx, mochaArgs)
 
+ ret._testConfig = _testConfig
  ret.originalTitle = originalTitle
 
  return ret
@@ -64,13 +65,11 @@ function overloadMochaFnForConfig (fnName, specWindow) {
  const origFn = subFn ? _fn[subFn] : _fn
 
  if (args.length > 2 && _.isObject(args[1])) {
- const opts = _.defaults({}, args[1], {
- browser: null,
- })
+ const _testConfig = _.extend({}, args[1])
 
  const mochaArgs = [args[0], args[2]]
 
- const configMatchesBrowser = opts.browser == null || Cypress.isBrowser(opts.browser, `${fnType} config value \`{ browser }\``)
+ const configMatchesBrowser = _testConfig.browser == null || Cypress.isBrowser(_testConfig.browser, `${fnType} config value \`{ browser }\``)
 
  if (!configMatchesBrowser) {
  // TODO: this would mess up the dashboard since it would be registered as a new test
@@ -84,15 +83,15 @@ function overloadMochaFnForConfig (fnName, specWindow) {
  this.skip()
  }
 
- return invokeFnWithOriginalTitle(this, originalTitle, mochaArgs, origFn)
+ return invokeFnWithOriginalTitle(this, originalTitle, mochaArgs, origFn, _testConfig)
  }
 
- return invokeFnWithOriginalTitle(this, originalTitle, mochaArgs, _fn['skip'])
+ return invokeFnWithOriginalTitle(this, originalTitle, mochaArgs, _fn['skip'], _testConfig)
  }
 
  const ret = origFn.apply(this, mochaArgs)
 
- ret.cfg = opts
+ ret._testConfig = _testConfig
 
  return ret
  }
@@ -327,7 +326,7 @@ function patchTestClone () {
  const ret = testClone.apply(this, arguments)
 
  // carry over testConfigOverrides
- ret.cfg = this.cfg
+ ret._testConfig = this._testConfig
 
  // carry over test.id
  ret.id = this.id

packages/driver/src/cypress/runner.js

@@ -9,6 +9,7 @@ const $Log = require('./log')
 const $utils = require('./utils')
 const $errUtils = require('./error_utils')
 const $stackUtils = require('./stack_utils')
+const { getResolvedTestConfigOverride } = require('../cy/testConfigOverrides')
 
 const mochaCtxKeysRe = /^(_runnable|test)$/
 const betweenQuotesRe = /\"(.+?)\"/
@@ -18,8 +19,7 @@ const TEST_BEFORE_RUN_EVENT = 'runner:test:before:run'
 const TEST_AFTER_RUN_EVENT = 'runner:test:after:run'
 
 const RUNNABLE_LOGS = 'routes agents commands hooks'.split(' ')
-const RUNNABLE_PROPS = 'id order title root hookName hookId err state failedFromHookId body speed type duration wallClockStartedAt wallClockDuration timings file originalTitle invocationDetails final currentRetry retries'.split(' ')
-
+const RUNNABLE_PROPS = '_testConfig id order title _titlePath root hookName hookId err state failedFromHookId body speed type duration wallClockStartedAt wallClockDuration timings file originalTitle invocationDetails final currentRetry retries'.split(' ')
 const debug = require('debug')('cypress:driver:runner')
 
 const fire = (event, runnable, Cypress) => {
@@ -493,6 +493,17 @@ const normalizeAll = (suite, initialTests = {}, setTestsById, setTests, onRunnab
  setTests(testsArr)
  }
 
+ // generate the diff of the config after spec has been executed
+ // e.g. config changes via Cypress.config('...')
+ normalizedSuite.runtimeConfig = {}
+ _.map(Cypress.config(), (v, key) => {
+ if (_.isEqual(v, Cypress.originalConfig[key])) {
+ return null
+ }
+
+ normalizedSuite.runtimeConfig[key] = v
+ })
+
  return normalizedSuite
 }
 
@@ -548,6 +559,17 @@ const normalize = (runnable, tests, initialTests, onRunnable, onLogsById, getRun
  // and collections
  const wrappedRunnable = wrapAll(runnable)
 
+ if (runnable.type === 'test') {
+ const cfg = getResolvedTestConfigOverride(runnable)
+
+ if (_.size(cfg)) {
+ runnable._testConfig = cfg
+ wrappedRunnable._testConfig = cfg
+ }
+
+ wrappedRunnable._titlePath = runnable.titlePath()
+ }
+
  if (prevAttempts) {
  wrappedRunnable.prevAttempts = prevAttempts
  }

packages/runner-ct/package.json

@@ -47,6 +47,7 @@
  "@babel/preset-env": "^7.12.1",
  "@packages/driver": "0.0.0-development",
  "@percy/cypress": "2.3.4",
+ "@types/node": "12.12.50",
  "@types/sockjs-client": "1.1.0",
  "babel-loader": "8.1.0",
  "clean-webpack-plugin": "^3.0.0",

packages/runner-ct/src/lib/event-manager.js

@@ -269,18 +269,20 @@ const eventManager = {
  Cypress.runner.setStartTime(state.startTime)
  }
 
+ if (config.isTextTerminal && !state.currentId) {
+ // we are in run mode and it's the first load
+ // store runnables in backend and maybe send to dashboard
+ return ws.emit('set:runnables:and:maybe:record:tests', runnables, run)
+ }
+
  if (state.currentId) {
  // if we have a currentId it means
  // we need to tell the Cypress to skip
  // ahead to that test
  Cypress.runner.resumeAtTest(state.currentId, state.emissions)
  }
 
- if (config.isTextTerminal && !state.currentId) {
- ws.emit('set:runnables', runnables, run)
- } else {
- run()
- }
+ run()
  })
  },
  })

packages/runner/cypress/support/helpers.js

@@ -260,7 +260,7 @@ function createCypress (defaultOptions = {}) {
  url: opts.visitUrl,
  } })
 
- .withArgs('set:runnables')
+ .withArgs('set:runnables:and:maybe:record:tests')
  .callsFake((...args) => {
  setRunnablesStub(...args)
  _.last(args)()

packages/runner/src/lib/event-manager.js

@@ -340,18 +340,20 @@ const eventManager = {
  Cypress.runner.setStartTime(state.startTime)
  }
 
+ if (config.isTextTerminal && !state.currentId) {
+ // we are in run mode and it's the first load
+ // store runnables in backend and maybe send to dashboard
+ return ws.emit('set:runnables:and:maybe:record:tests', runnables, run)
+ }
+
  if (state.currentId) {
  // if we have a currentId it means
  // we need to tell the Cypress to skip
  // ahead to that test
  Cypress.runner.resumeAtTest(state.currentId, state.emissions)
  }
 
- if (config.isTextTerminal && !state.currentId) {
- ws.emit('set:runnables', runnables, run)
- } else {
- run()
- }
+ run()
  })
  },
  })