From 46bf1ef349182814cf1591b3cd733547fc77fc58 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Tue, 1 Feb 2022 10:22:44 -0600 Subject: [PATCH 01/64] Adding remaining reporter error tests. --- .../cypress/e2e/runner/reporter.errors.cy.ts | 992 +++++++++++++++++- .../{verify-helpers.ts => verify-failures.ts} | 31 +- .../cypress/e2e/errors/commands.cy.js | 9 + .../cypress/e2e/errors/custom_commands.cy.js | 25 + .../cypress/e2e/errors/docs_url.cy.js | 5 + .../cypress/e2e/errors/each.cy.js | 19 + .../cypress/e2e/errors/events.cy.js | 33 + .../cypress/e2e/errors/exceptions.cy.js | 11 + .../cypress/e2e/errors/hooks.cy.js | 18 + .../cypress/e2e/errors/intercept.cy.ts | 114 ++ .../cypress/e2e/errors/nested_hooks.cy.js | 19 + .../cypress/e2e/errors/readfile.cy.js | 5 + .../cypress/e2e/errors/route.cy.js | 100 ++ .../cypress/e2e/errors/server.cy.js | 80 ++ .../cypress/e2e/errors/should.cy.js | 56 + .../cypress/e2e/errors/spread.cy.js | 19 + .../cypress/e2e/errors/then.cy.js | 19 + .../cypress/e2e/errors/typescript.cy.ts | 22 + .../cypress/e2e/errors/uncaught.cy.js | 79 ++ .../e2e/errors/uncaught_outside_test.cy.js | 5 + .../uncaught_outside_test_only_suite.cy.js | 12 + .../cypress/e2e/errors/unexpected.cy.js | 26 + .../cypress/e2e/errors/validation.cy.js | 13 + .../cypress/e2e/errors/visit.cy.js | 33 + .../cypress/e2e/errors/within.cy.js | 19 + .../cypress/e2e/errors/wrap.cy.js | 19 + .../cypress/fixtures/errors.html | 69 ++ .../cypress/fixtures/index.html | 3 + .../runner-e2e-specs/src/throws-error.js | 6 + .../projects/runner-e2e-specs/tsconfig.json | 16 + 30 files changed, 1818 insertions(+), 59 deletions(-) rename packages/app/cypress/e2e/runner/support/{verify-helpers.ts => verify-failures.ts} (86%) create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/commands.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/custom_commands.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/docs_url.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/each.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/events.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/exceptions.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/hooks.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/intercept.cy.ts create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/nested_hooks.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/readfile.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/route.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/server.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/should.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/spread.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/then.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/typescript.cy.ts create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/uncaught.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/uncaught_outside_test.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/uncaught_outside_test_only_suite.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/unexpected.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/validation.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/visit.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/within.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/errors/wrap.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/fixtures/errors.html create mode 100644 system-tests/projects/runner-e2e-specs/cypress/fixtures/index.html create mode 100644 system-tests/projects/runner-e2e-specs/src/throws-error.js create mode 100644 system-tests/projects/runner-e2e-specs/tsconfig.json diff --git a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts index 30f070e042f4..636d31423afb 100644 --- a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts +++ b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts @@ -1,39 +1,49 @@ -import { verify } from './support/verify-helpers' +import { verify, verifyInternalFailure } from './support/verify-failures' -describe('errors ui', { - viewportHeight: 768, - viewportWidth: 1024, -}, () => { - describe('assertion failures', () => { - beforeEach(() => { - cy.scaffoldProject('runner-e2e-specs') - cy.openProject('runner-e2e-specs') +const setup = ({ fileName, mockPreferredEditor, onLoadStatsMessage }) => { + cy.scaffoldProject('runner-e2e-specs') + cy.openProject('runner-e2e-specs') - // set preferred editor to bypass IDE selection dialog - cy.withCtx((ctx) => { - ctx.coreData.localSettings.availableEditors = [ - ...ctx.coreData.localSettings.availableEditors, - { - id: 'test-editor', - binary: '/usr/bin/test-editor', - name: 'Test editor', - }, - ] + if (mockPreferredEditor) { + // set preferred editor to bypass IDE selection dialog + cy.withCtx((ctx) => { + ctx.coreData.localSettings.availableEditors = [ + ...ctx.coreData.localSettings.availableEditors, + { + id: 'test-editor', + binary: '/usr/bin/test-editor', + name: 'Test editor', + }, + ] - ctx.coreData.localSettings.preferences.preferredEditorBinary = 'test-editor' - }) + ctx.coreData.localSettings.preferences.preferredEditorBinary = 'test-editor' + }) + } - cy.startAppServer() - cy.visitApp() + cy.startAppServer() + cy.visitApp() - cy.contains('[data-cy=spec-item]', 'assertions.cy.js').click() + cy.contains('[data-cy=spec-item]', fileName).click() - cy.location().should((location) => { - expect(location.hash).to.contain('assertions.cy.js') - }) + cy.location().should((location) => { + expect(location.hash).to.contain(fileName) + }) + + // Wait for specs to complete + cy.findByLabelText('Stats').get('.failed', { timeout: 10000 }).should('have.text', onLoadStatsMessage) +} - // Wait for specs to complete - cy.findByLabelText('Stats').get('.failed', { timeout: 10000 }).should('have.text', 'Failed:3') +describe('errors ui', { + viewportHeight: 768, + viewportWidth: 1024, +}, () => { + describe('assertion failures', () => { + before(() => { + setup({ + fileName: 'assertions.cy.js', + mockPreferredEditor: true, + onLoadStatsMessage: 'Failed:3', + }) }) verify.it('with expect().', { @@ -41,7 +51,6 @@ describe('errors ui', { hasPreferredIde: true, column: 25, message: `expected 'actual' to equal 'expected'`, - codeFrameText: 'with expect().', }) verify.it('with assert()', { @@ -49,7 +58,6 @@ describe('errors ui', { hasPreferredIde: true, column: '(5|12)', // (chrome|firefox) message: `should be true`, - codeFrameText: 'with assert()', }) verify.it('with assert.()', { @@ -57,26 +65,16 @@ describe('errors ui', { hasPreferredIde: true, column: 12, message: `expected 'actual' to equal 'expected'`, - codeFrameText: 'with assert.()', }) }) describe('assertion failures - no preferred IDE', () => { - beforeEach(() => { - cy.scaffoldProject('runner-e2e-specs') - cy.openProject('runner-e2e-specs') - - cy.startAppServer() - cy.visitApp() - - cy.contains('[data-cy=spec-item]', 'assertions.cy.js').click() - - cy.location().should((location) => { - expect(location.hash).to.contain('assertions.cy.js') + before(() => { + setup({ + fileName: 'assertions.cy.js', + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:3', }) - - // Wait for specs to complete - cy.findByLabelText('Stats').get('.failed', { timeout: 10000 }).should('have.text', 'Failed:3') }) verify.it('with expect().', { @@ -87,4 +85,906 @@ describe('errors ui', { codeFrameText: 'with expect().', }) }) + + describe('exception failures', () => { + before(() => { + setup({ + fileName: 'exceptions.cy.js', + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:2', + }) + }) + + verify.it('in spec file', { + file: 'exceptions.cy.js', + column: 10, + message: 'bar is not a function', + }) + + verify.it('in file outside project', { + file: 'exceptions.cy.js', + message: 'An outside error', + regex: /\/throws\-error\.js:5:9/, + codeFrameText: `thrownewError('An outside error')`, + verifyOpenInIde: false, + }) + }) + + describe('hooks', { viewportHeight: 900 }, () => { + before(() => { + setup({ + fileName: 'hooks.cy.js', + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:1', + }) + }) + + // https://github.com/cypress-io/cypress/issues/8214 + // https://github.com/cypress-io/cypress/issues/8288 + // https://github.com/cypress-io/cypress/issues/8350 + verify.it('errors when a hook is nested in another hook', { + file: 'hooks.cy.js', + specTitle: 'test', + column: '(7|18)', // (chrome|firefox) + codeFrameText: 'beforeEach(()=>', + message: `Cypress detected you registered a(n) beforeEach hook while a test was running`, + }) + }) + + describe('commands', () => { + before(() => { + setup({ + fileName: 'commands.cy.js', + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:2', + }) + }) + + verify.it('failure', { + file: 'commands.cy.js', + column: 8, + message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', + }) + + verify.it('chained failure', { + file: 'commands.cy.js', + column: 20, + message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', + }) + }) + + describe('cy.should', () => { + const file = 'should.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:8', + }) + }) + + verify.it('callback assertion failure', { + file, + column: 27, + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('callback exception', { + file, + column: 12, + message: 'bar is not a function', + }) + + verify.it('standard assertion failure', { + file, + column: 6, + message: 'Timed out retrying after 0ms: expected {} to have property \'foo\'', + }) + + verify.it('after multiple', { + file, + column: 6, + message: 'Timed out retrying after 0ms: expected \'foo\' to equal \'bar\'', + }) + + verify.it('after multiple callbacks exception', { + file, + column: 12, + codeFrameText: '({}).bar()', + message: 'bar is not a function', + }) + + verify.it('after multiple callbacks assertion failure', { + file, + column: 27, + codeFrameText: '.should(()=>', + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('after callback success assertion failure', { + file, + column: 6, + codeFrameText: '.should(\'have.property', + message: `expected {} to have property 'foo'`, + }) + + verify.it('command failure after success', { + file, + column: 8, + message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', + }) + }) + + describe('cy.each', () => { + const file = 'each.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:3', + }) + }) + + verify.it('assertion failure', { + file, + column: 27, + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('exception', { + file, + column: 12, + message: 'bar is not a function', + }) + + verify.it('command failure', { + file, + column: 10, + message: 'Expected to find element: #does-not-exist, but never found it', + }) + }) + + describe('cy.spread', () => { + const file = 'spread.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:3', + }) + }) + + verify.it('assertion failure', { + file, + column: 27, + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('exception', { + file, + column: 12, + message: 'bar is not a function', + }) + + verify.it('command failure', { + file, + column: 10, + message: 'Expected to find element: #does-not-exist, but never found it', + }) + }) + + describe('cy.within', () => { + const file = 'within.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:3', + }) + }) + + verify.it('assertion failure', { + file, + column: 27, + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('exception', { + file, + column: 12, + message: 'bar is not a function', + }) + + verify.it('command failure', { + file, + column: 10, + message: 'Expected to find element: #does-not-exist, but never found it', + }) + }) + + describe('cy.wrap', () => { + const file = 'wrap.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:3', + }) + }) + + verify.it('assertion failure', { + file, + column: 27, + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('exception', { + file, + column: 12, + message: 'bar is not a function', + }) + + verify.it('command failure', { + file, + column: 10, + message: 'Expected to find element: #does-not-exist, but never found it', + }) + }) + + describe('cy.visit', () => { + const file = 'visit.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:4', + }) + }) + + verify.it('onBeforeLoad assertion failure', { + file, + column: 29, + codeFrameText: 'onBeforeLoad', + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('onBeforeLoad exception', { + file, + column: 14, + codeFrameText: 'onBeforeLoad', + message: 'bar is not a function', + }) + + verify.it('onLoad assertion failure', { + file, + column: 29, + codeFrameText: 'onLoad', + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('onLoad exception', { + file, + column: 14, + codeFrameText: 'onLoad', + message: 'bar is not a function', + }) + }) + + describe('cy.intercept', () => { + const file = 'intercept.cy.ts' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:3', + }) + }) + + verify.it('assertion failure in req callback', { + file, + column: 22, + message: [ + `expected 'a' to equal 'b'`, + ], + notInMessage: [ + 'The following error originated from your spec code', + ], + }) + + verify.it('assertion failure in res callback', { + file, + column: 24, + codeFrameText: '.reply(()=>{', + message: [ + `expected 'b' to equal 'c'`, + ], + notInMessage: [ + 'The following error originated from your spec code', + ], + }) + + verify.it('fails when erroneous response is received while awaiting response', { + file, + column: 6, + // this fails the active test because it's an asynchronous + // response failure from the network + codeFrameText: '83|.then(()=>{', + message: [ + 'A callback was provided to intercept the upstream response, but a network error occurred while making the request', + ], + notInMessage: [ + 'The following error originated from your spec code', + ], + }) + }) + + describe('cy.route', () => { + const file = 'route.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:9', + }) + }) + + verify.it('callback assertion failure', { + file, + column: 27, + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('callback exception', { + file, + column: 12, + message: 'bar is not a function', + }) + + verify.it('command failure', { + file, + column: 10, + message: 'Expected to find element: #does-not-exist, but never found it', + }) + + verify.it('onAbort assertion failure', { + file, + column: 29, + codeFrameText: 'onAbort', + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('onAbort exception', { + file, + column: 14, + codeFrameText: 'onAbort', + message: 'bar is not a function', + }) + + verify.it('onRequest assertion failure', { + file, + column: 29, + codeFrameText: 'onRequest', + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('onRequest exception', { + file, + column: 14, + codeFrameText: 'onRequest', + message: 'bar is not a function', + }) + + verify.it('onResponse assertion failure', { + file, + column: 29, + codeFrameText: 'onResponse', + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('onResponse exception', { + file, + column: 14, + codeFrameText: 'onResponse', + message: 'bar is not a function', + }) + }) + + describe('cy.server', () => { + const file = 'server.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:6', + }) + }) + + verify.it('onAbort assertion failure', { + file, + column: 29, + codeFrameText: 'onAbort', + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('onAbort exception', { + file, + column: 14, + codeFrameText: 'onAbort', + message: 'bar is not a function', + }) + + verify.it('onRequest assertion failure', { + file, + column: 29, + codeFrameText: 'onRequest', + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('onRequest exception', { + file, + column: 14, + codeFrameText: 'onRequest', + message: 'bar is not a function', + }) + + verify.it('onResponse assertion failure', { + file, + column: 29, + codeFrameText: 'onResponse', + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('onResponse exception', { + file, + column: 14, + codeFrameText: 'onResponse', + message: 'bar is not a function', + }) + }) + + describe('cy.readFile', () => { + const file = 'readfile.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:1', + }) + }) + + verify.it('existence failure', { + file, + column: 8, + message: 'failed because the file does not exist', + }) + }) + + describe('validation errors', () => { + const file = 'validation.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:3', + }) + }) + + verify.it('from cypress', { + file, + column: 8, + message: 'can only accept a string preset or', + stack: ['throwErrBadArgs', 'From Your Spec Code:'], + }) + + verify.it('from chai expect', { + file, + column: '(5|12)', // (chrome|firefox) + message: 'Invalid Chai property: nope', + stack: ['proxyGetter', 'From Your Spec Code:'], + }) + + verify.it('from chai assert', { + file, + column: 12, + message: 'object tested must be an array', + }) + }) + + describe('event handlers', () => { + const file = 'events.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:3', + }) + }) + + verify.it('event assertion failure', { + file, + column: 27, + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('event exception', { + file, + column: 12, + message: 'bar is not a function', + }) + + verify.it('fail handler assertion failure', { + file, + column: 27, + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('fail handler exception', { + file, + column: 12, + message: 'bar is not a function', + }) + }) + + describe('uncaught errors', () => { + const file = 'uncaught.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:11', + }) + }) + + verify.it('sync app visit exception', { + file, + uncaught: true, + command: 'visit', + originalMessage: 'visit error', + message: [ + 'The following error originated from your application code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + regex: /localhost\:\d+\/cypress\/fixtures\/errors.html\?error-on-visit:\d+:\d+/, + hasCodeFrame: false, + verifyOpenInIde: false, + }) + + verify.it('sync app navigates to visit exception', { + file, + uncaught: true, + originalMessage: 'visit error', + message: [ + 'The following error originated from your application code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + regex: /localhost\:\d+\/cypress\/fixtures\/errors.html\?error-on-visit:\d+:\d+/, + hasCodeFrame: false, + verifyOpenInIde: false, + }) + + verify.it('sync app exception', { + file, + uncaught: true, + command: 'click', + originalMessage: 'sync error', + message: [ + 'The following error originated from your application code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + regex: /localhost\:\d+\/cypress\/fixtures\/errors.html:\d+:\d+/, + hasCodeFrame: false, + verifyOpenInIde: false, + }) + + verify.it('async app exception', { + file, + uncaught: true, + originalMessage: 'async error', + message: [ + 'The following error originated from your application code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + regex: /localhost\:\d+\/cypress\/fixtures\/errors.html:\d+:\d+/, + hasCodeFrame: false, + verifyOpenInIde: false, + }) + + verify.it('app unhandled rejection', { + file, + uncaught: true, + originalMessage: 'promise rejection', + message: [ + 'The following error originated from your application code', + 'It was caused by an unhandled promise rejection', + ], + regex: /localhost\:\d+\/cypress\/fixtures\/errors.html:\d+:\d+/, + hasCodeFrame: false, + verifyOpenInIde: false, + }) + + verify.it('async spec exception', { + file, + uncaught: true, + column: 12, + originalMessage: 'bar is not a function', + message: [ + 'The following error originated from your test code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + }) + + verify.it('async spec exception with done', { + file, + uncaught: true, + column: 12, + originalMessage: 'bar is not a function', + message: [ + 'The following error originated from your test code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + }) + + verify.it('spec unhandled rejection', { + file, + uncaught: true, + column: 20, + originalMessage: 'Unhandled promise rejection from the spec', + message: [ + 'The following error originated from your test code', + 'It was caused by an unhandled promise rejection', + ], + }) + + verify.it('spec unhandled rejection with done', { + file, + uncaught: true, + column: 20, + originalMessage: 'Unhandled promise rejection from the spec', + message: [ + 'The following error originated from your test code', + 'It was caused by an unhandled promise rejection', + ], + }) + + verify.it('spec Bluebird unhandled rejection', { + file, + uncaught: true, + column: 21, + originalMessage: 'Unhandled promise rejection from the spec', + message: [ + 'The following error originated from your test code', + 'It was caused by an unhandled promise rejection', + ], + }) + + verify.it('spec Bluebird unhandled rejection with done', { + file, + uncaught: true, + column: 21, + originalMessage: 'Unhandled promise rejection from the spec', + message: [ + 'The following error originated from your test code', + 'It was caused by an unhandled promise rejection', + ], + }) + + verify.it('exception inside uncaught:exception', { + file, + uncaught: true, + uncaughtMessage: 'sync error', + column: 12, + originalMessage: 'bar is not a function', + message: [ + 'The following error originated from your test code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + }) + }) + + describe('uncaught errors: outside test', () => { + const file = 'uncaught_outside_test.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:1', + }) + }) + + // NOTE: the following 2 test don't have uncaught: true because we don't + // display command logs if there are only events and not true commands + // and uncaught: true causes the verification to look for the error + // event command log + verify.it('spec exception outside test', { + file, + column: 7, + specTitle: 'An uncaught error was detected outside of a test', + message: [ + 'The following error originated from your test code', + 'error from outside test', + 'Cypress could not associate this error to any specific test', + ], + codeFrameText: `thrownewError('error from outside test')`, + }) + }) + + describe('uncaught errors: outside test only suite', () => { + const file = 'uncaught_outside_test_only_suite.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:1', + }) + }) + + verify.it('spec exception outside test with only suite', { + file, + column: 7, + specTitle: 'An uncaught error was detected outside of a test', + message: [ + 'error from outside test with only suite', + 'The following error originated from your test code', + 'Cypress could not associate this error to any specific test', + ], + codeFrameText: `thrownewError('error from outside test with only suite')`, + }) + }) + + describe('custom commands', () => { + const file = 'custom_commands.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:2', + }) + }) + + verify.it('assertion failure', { + file, + column: 23, + message: `expected 'actual' to equal 'expected'`, + codeFrameText: `add('failAssertion'`, + }) + + verify.it('exception', { + file, + column: 8, + message: 'bar is not a function', + codeFrameText: `add('failException'`, + }) + + verify.it('command failure', { + file, + column: 6, + message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', + codeFrameText: `add('failCommand'`, + }) + }) + + describe('typescript', () => { + const file = 'typescript.cy.ts' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:3', + }) + }) + + verify.it('assertion failure', { + file, + column: 25, + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('exception', { + file, + column: 10, + message: 'bar is not a function', + }) + + verify.it('command failure', { + file, + column: 8, + message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', + }) + }) + + describe('docs url', () => { + const file = 'docs_url.cy.js' + const docsUrl = 'https://on.cypress.io/viewport' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:1', + }) + }) + + verify.it('displays doc url', { retries: 1 }, { + file, + verifyFn () { + cy + .contains('.runnable-title', 'displays doc url') + .closest('.runnable').within(() => { + if (Cypress.config('isInteractive')) { + // renders href in open mode + cy.get('.runnable-err-message') + .should('not.contain', docsUrl) + .contains('Learn more') + .should('have.attr', 'href', docsUrl) + } else { + // renders link as text in run mode + cy.get('.runnable-err-message') + .should('contain', docsUrl) + .contains('Learn more') + .should('not.exist') + } + }) + }, + }) + }) + + // cases where there is a bug in Cypress and we should show cypress internals + // instead of the invocation stack. we test this by monkey-patching internal + // methods to make them throw an error + describe('unexpected errors', () => { + const file = 'unexpected.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:1', + }) + }) + + // FIXME: the eval doesn't seem to take effect and overwrite the method + // so it ends up not failing properly + verify.it.skip('Cypress method error', { + file, + verifyFn: verifyInternalFailure, + method: 'Cypress.LocalStorage._isSpecialKeyword', + }) + + verify.it('internal cy error', { + file, + verifyFn: verifyInternalFailure, + method: 'cy.expect', + }) + }) }) diff --git a/packages/app/cypress/e2e/runner/support/verify-helpers.ts b/packages/app/cypress/e2e/runner/support/verify-failures.ts similarity index 86% rename from packages/app/cypress/e2e/runner/support/verify-helpers.ts rename to packages/app/cypress/e2e/runner/support/verify-failures.ts index dd0a742f9beb..716f8a8aa546 100644 --- a/packages/app/cypress/e2e/runner/support/verify-helpers.ts +++ b/packages/app/cypress/e2e/runner/support/verify-failures.ts @@ -27,7 +27,6 @@ export const verifyFailure = (options) => { verifyOpenInIde = true, hasPreferredIde, column, - codeFrameText, originalMessage, message = [], notInMessage = [], @@ -37,7 +36,11 @@ export const verifyFailure = (options) => { uncaught = false, uncaughtMessage, } = options - let { regex, line } = options + let { regex, line, codeFrameText } = options + + if (!codeFrameText) { + codeFrameText = specTitle + } regex = regex || new RegExp(`${file}:${line || '\\d+'}:${column}`) @@ -174,7 +177,7 @@ const createVerifyTest = (modifier?: string) => { props.specTitle ||= title - const verifyFn = props.verifyFn || verifyFailure.bind(null, props) + const verifyFn = (props.verifyFn || verifyFailure).bind(null, props) return (modifier ? it[modifier] : it)(title, verifyFn) } @@ -188,18 +191,20 @@ verify.it['only'] = createVerifyTest('only') verify.it['skip'] = createVerifyTest('skip') export const verifyInternalFailure = (props) => { - const { method, stackMethod } = props + const { specTitle, method, stackMethod } = props - cy.get('.runnable-err-message') - .should('include.text', `thrown in ${method.replace(/\./g, '-')}`) + cy.contains('.runnable-title', specTitle).closest('.runnable').within(() => { + cy.get('.runnable-err-message') + .should('include.text', `thrown in ${method.replace(/\./g, '-')}`) - cy.get('.runnable-err-stack-expander > .collapsible-header').click() + cy.get('.runnable-err-stack-expander > .collapsible-header').click() - cy.get('.runnable-err-stack-trace') - .should('include.text', stackMethod || method) + cy.get('.runnable-err-stack-trace') + .should('include.text', stackMethod || method) - // this is an internal cypress error and we can only show code frames - // from specs, so it should not show the code frame - cy.get('.test-err-code-frame') - .should('not.exist') + // this is an internal cypress error and we can only show code frames + // from specs, so it should not show the code frame + cy.get('.test-err-code-frame') + .should('not.exist') + }) } diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/commands.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/commands.cy.js new file mode 100644 index 000000000000..69fbad12d33b --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/commands.cy.js @@ -0,0 +1,9 @@ +describe('commands', { defaultCommandTimeout: 0 }, () => { + it('failure', () => { + cy.get('#does-not-exist') + }) + + it('chained failure', () => { + cy.get('body').find('#does-not-exist') + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/custom_commands.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/custom_commands.cy.js new file mode 100644 index 000000000000..a5e35ee3c040 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/custom_commands.cy.js @@ -0,0 +1,25 @@ +Cypress.Commands.add('failAssertion', () => { + expect('actual').to.equal('expected') +}) + +Cypress.Commands.add('failException', () => { + ({}).bar() +}) + +Cypress.Commands.add('failCommand', () => { + cy.get('#does-not-exist') +}) + +describe('custom commands', { defaultCommandTimeout: 0 }, () => { + it('assertion failure', () => { + cy.failAssertion() + }) + + it('exception', () => { + cy.failException() + }) + + it('command failure', () => { + cy.failCommand() + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/docs_url.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/docs_url.cy.js new file mode 100644 index 000000000000..eaf93f9cbb26 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/docs_url.cy.js @@ -0,0 +1,5 @@ +describe('docs url', () => { + it('displays doc url', () => { + cy.viewport() + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/each.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/each.cy.js new file mode 100644 index 000000000000..2059c9701b60 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/each.cy.js @@ -0,0 +1,19 @@ +describe('cy.each', { defaultCommandTimeout: 0 }, () => { + it('assertion failure', () => { + cy.wrap([1]).each(() => { + expect('actual').to.equal('expected') + }) + }) + + it('exception', () => { + cy.wrap([1]).each(() => { + ({}).bar() + }) + }) + + it('command failure', () => { + cy.wrap([1]).each(() => { + cy.get('#does-not-exist') + }) + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/events.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/events.cy.js new file mode 100644 index 000000000000..1321c7465c74 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/events.cy.js @@ -0,0 +1,33 @@ +describe('event handlers', { defaultCommandTimeout: 0 }, () => { + it('event assertion failure', () => { + cy.on('window:load', () => { + expect('actual').to.equal('expected') + }) + + cy.visit('http://localhost:1919') + }) + + it('event exception', () => { + cy.on('window:load', () => { + ({}).bar() + }) + + cy.visit('http://localhost:1919') + }) + + it('fail handler assertion failure', () => { + cy.on('fail', () => { + expect('actual').to.equal('expected') + }) + + cy.get('#does-not-exist') + }) + + it('fail handler exception', () => { + cy.on('fail', () => { + ({}).bar() + }) + + cy.get('#does-not-exist') + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/exceptions.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/exceptions.cy.js new file mode 100644 index 000000000000..ca7379ab44ee --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/exceptions.cy.js @@ -0,0 +1,11 @@ +const outsideError = require('../../../src/throws-error') + +describe('exception failures', () => { + it('in spec file', () => { + ({}).bar() + }) + + it('in file outside project', () => { + outsideError() + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/hooks.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/hooks.cy.js new file mode 100644 index 000000000000..4da8922fee66 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/hooks.cy.js @@ -0,0 +1,18 @@ +const log = () => { + const r = cy.state('runnable') + + assert(true, `${r.type} - ${r.parent.title || 'root'}`) +} + +describe('nested_hooks', () => { + describe('nested beforeEach', () => { + before(() => { + log() + beforeEach(() => { + log() + }) + }) + + it('test', log) + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/intercept.cy.ts b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/intercept.cy.ts new file mode 100644 index 000000000000..6f1b964a5e3f --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/intercept.cy.ts @@ -0,0 +1,114 @@ +describe('cy.intercept', () => { + const emitProxyLog = () => { + return Cypress.emit('request:event', 'incoming:request', { + requestId: 1, + method: 'GET', + url: '', + headers: {}, + resourceType: 'other', + originalResourceType: 'other', + }) + } + + it('assertion failure in req callback', () => { + cy.intercept('/json-content-type', () => { + expect('a').to.eq('b') + }) + .then(() => { + emitProxyLog() + Cypress.emit('net:stubbing:event', 'before:request', { + browserRequestId: 1, + eventId: '1', + subscription: { + // @ts-ignore + routeId: Object.keys(Cypress.state('routes'))[0], + await: true, + }, + data: { + url: '', + }, + }) + }) + .wait(1000) // ensure the failure happens before test ends + }) + + it('assertion failure in res callback', () => { + cy.intercept('/json-content-type', (req) => { + req.reply(() => { + expect('b').to.eq('c') + }) + }) + .then(() => { + emitProxyLog() + Cypress.emit('net:stubbing:event', 'before:request', { + browserRequestId: 1, + eventId: '1', + requestId: '1', + subscription: { + // @ts-ignore + routeId: Object.keys(Cypress.state('routes'))[0], + await: true, + }, + data: { + url: '', + }, + }) + + Cypress.emit('net:stubbing:event', 'before:response', { + eventId: '1', + requestId: '1', + subscription: { + // @ts-ignore + id: Object.values(Cypress.state('routes'))[0].requests['1'].subscriptions[0].subscription.id, + // @ts-ignore + routeId: Object.keys(Cypress.state('routes'))[0], + await: true, + }, + data: { + url: '', + }, + }) + }) + .wait(1000) // ensure the failure happens before test ends + }) + + it('fails when erroneous response is received while awaiting response', () => { + cy.intercept('/fake', (req) => { + req.reply(() => { + throw new Error('this should not be reached') + }) + }) + .then(() => { + emitProxyLog() + Cypress.emit('net:stubbing:event', 'before:request', { + browserRequestId: 1, + eventId: '1', + requestId: '1', + subscription: { + // @ts-ignore + routeId: Object.keys(Cypress.state('routes'))[0], + await: true, + }, + data: { + url: '', + }, + }) + + Cypress.emit('net:stubbing:event', 'network:error', { + eventId: '1', + requestId: '1', + subscription: { + // @ts-ignore + routeId: Object.keys(Cypress.state('routes'))[0], + }, + data: { + error: { + name: 'ResponseError', + message: 'it errored', + }, + }, + }) + }) + .wait(1000) // ensure the failure happens before test ends + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/nested_hooks.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/nested_hooks.cy.js new file mode 100644 index 000000000000..74795593ac53 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/nested_hooks.cy.js @@ -0,0 +1,19 @@ +/// +const log = function () { + const r = cy.state('runnable') + + assert(true, `${r.type} - ${r.parent.title || 'root'}`) +} + +describe('nested_hooks', () => { + describe('nested beforeEach', () => { + before(() => { + log() + beforeEach(() => { + log() + }) + }) + + it('test', log) + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/readfile.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/readfile.cy.js new file mode 100644 index 000000000000..f682bd528d58 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/readfile.cy.js @@ -0,0 +1,5 @@ +describe('cy.readFile', () => { + it('existence failure', () => { + cy.readFile('does-not-exist', { timeout: 0 }) + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/route.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/route.cy.js new file mode 100644 index 000000000000..92c1c1ce9f7e --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/route.cy.js @@ -0,0 +1,100 @@ +// eslint-disable-next-line +export const sendXhr = (route) => (win) => { + const xhr = new win.XMLHttpRequest() + + xhr.open('GET', route) + xhr.send() + + return xhr +} + +// eslint-disable-next-line +export const abortXhr = (route) => (win) => { + const xhr = sendXhr(route)(win) + + xhr.abort() +} + +describe('cy.route', { defaultCommandTimeout: 0 }, () => { + it('callback assertion failure', () => { + cy.server().route(() => { + expect('actual').to.equal('expected') + }) + }) + + it('callback exception', () => { + cy.server().route(() => { + ({}).bar() + }) + }) + + it('command failure', () => { + cy.server().route(() => { + cy.get('#does-not-exist') + + return '/foo' + }) + }) + + it('onAbort assertion failure', () => { + cy.server().route({ + url: '/foo', + onAbort () { + expect('actual').to.equal('expected') + }, + }) + .window().then(abortXhr('/foo')) + }) + + it('onAbort exception', () => { + cy.server().route({ + url: '/foo', + onAbort () { + ({}).bar() + }, + }) + .window().then(abortXhr('/foo')) + }) + + it('onRequest assertion failure', () => { + cy.server().route({ + url: '/foo', + onRequest () { + expect('actual').to.equal('expected') + }, + }) + .window().then(sendXhr('/foo')) + }) + + it('onRequest exception', () => { + cy.server().route({ + url: '/foo', + onRequest () { + ({}).bar() + }, + }) + .window().then(sendXhr('/foo')) + }) + + it('onResponse assertion failure', () => { + cy.server().route({ + url: '/json-content-type', + onResponse () { + expect('actual').to.equal('expected') + }, + }) + .window().then(sendXhr('/json-content-type')) + .wait(10000) + }) + + it('onResponse exception', () => { + cy.server().route({ + url: '/json-content-type', + onResponse () { + ({}).bar() + }, + }) + .window().then(sendXhr('/json-content-type')) + .wait(10000) + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/server.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/server.cy.js new file mode 100644 index 000000000000..cf7c7e8aec28 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/server.cy.js @@ -0,0 +1,80 @@ +// eslint-disable-next-line +export const sendXhr = (route) => (win) => { + const xhr = new win.XMLHttpRequest() + + xhr.open('GET', route) + xhr.send() + + return xhr +} + +// eslint-disable-next-line +export const abortXhr = (route) => (win) => { + const xhr = sendXhr(route)(win) + + xhr.abort() +} + +describe('cy.server', { defaultCommandTimeout: 0 }, () => { + it('onAbort assertion failure', () => { + cy.server({ + onAbort () { + expect('actual').to.equal('expected') + }, + }) + .route('/foo') + .window().then(abortXhr('/foo')) + }) + + it('onAbort exception', () => { + cy.server({ + onAbort () { + ({}).bar() + }, + }) + .route('/foo') + .window().then(abortXhr('/foo')) + }) + + it('onRequest assertion failure', () => { + cy.server({ + onRequest () { + expect('actual').to.equal('expected') + }, + }) + .route('/foo') + .window().then(sendXhr('/foo')) + }) + + it('onRequest exception', () => { + cy.server({ + onRequest () { + ({}).bar() + }, + }) + .route('/foo') + .window().then(sendXhr('/foo')) + }) + + it('onResponse assertion failure', () => { + cy.server({ + onResponse () { + expect('actual').to.equal('expected') + }, + }) + .route('/json-content-type') + .window().then(sendXhr('/json-content-type')) + .wait(10000) + }) + + it('onResponse exception', () => { + cy.server({ + onResponse () { + ({}).bar() + }, + }) + .route('/json-content-type') + .window().then(sendXhr('/json-content-type')) + .wait(10000) + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/should.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/should.cy.js new file mode 100644 index 000000000000..18b12347b677 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/should.cy.js @@ -0,0 +1,56 @@ +describe('cy.should', { defaultCommandTimeout: 0 }, () => { + it('callback assertion failure', () => { + cy.wrap({}).should(() => { + expect('actual').to.equal('expected') + }) + }) + + it('callback exception', () => { + cy.wrap({}).should(() => { + ({}).bar() + }) + }) + + it('standard assertion failure', () => { + cy.wrap({}) + .should('have.property', 'foo') + }) + + it('after multiple', () => { + cy.wrap({ foo: 'foo' }).should('have.property', 'foo') + .should('equal', 'bar') + }) + + it('after multiple callbacks exception', () => { + cy.wrap({ foo: 'foo' }) + .should(() => { + expect(true).to.be.true + }) + .should(() => { + ({}).bar() + }) + }) + + it('after multiple callbacks assertion failure', () => { + cy.wrap({ foo: 'foo' }) + .should(() => { + expect(true).to.be.true + }) + .should(() => { + expect('actual').to.equal('expected') + }) + }) + + it('after callback success assertion failure', () => { + cy.wrap({}) + .should(() => { + expect(true).to.be.true + }) + .should('have.property', 'foo') + }) + + it('command failure after success', () => { + cy.wrap({ foo: 'foo' }).should('have.property', 'foo') + cy.get('#does-not-exist') + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/spread.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/spread.cy.js new file mode 100644 index 000000000000..c3b407af458d --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/spread.cy.js @@ -0,0 +1,19 @@ +describe('cy.spread', { defaultCommandTimeout: 0 }, () => { + it('assertion failure', () => { + cy.wrap([1, 2, 3]).spread(() => { + expect('actual').to.equal('expected') + }) + }) + + it('exception', () => { + cy.wrap([1, 2, 3]).spread(() => { + ({}).bar() + }) + }) + + it('command failure', () => { + cy.wrap([1, 2, 3]).spread(() => { + cy.get('#does-not-exist') + }) + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/then.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/then.cy.js new file mode 100644 index 000000000000..d995f0a53dfe --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/then.cy.js @@ -0,0 +1,19 @@ +describe('cy.then', { defaultCommandTimeout: 0 }, () => { + it('assertion failure', () => { + cy.wrap({}).then(() => { + expect('actual').to.equal('expected') + }) + }) + + it('exception', () => { + cy.wrap({}).then(() => { + ({}).bar() + }) + }) + + it('command failure', () => { + cy.wrap({}).then(() => { + cy.get('#does-not-exist') + }) + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/typescript.cy.ts b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/typescript.cy.ts new file mode 100644 index 000000000000..4ba57ae4f2bc --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/typescript.cy.ts @@ -0,0 +1,22 @@ +// simple example of typescript types +type Foo = { + something: string +} + +describe('typescript', { defaultCommandTimeout: 0 }, () => { + it('assertion failure', () => { + expect('actual').to.equal('expected') + }) + + it('exception', () => { + // @ts-ignore + ({}).bar() + }) + + it('command failure', () => { + const myVar = { something: 'foo' } as Foo + + cy.log(`Printing TS: ${ myVar}`) + cy.get('#does-not-exist') + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/uncaught.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/uncaught.cy.js new file mode 100644 index 000000000000..8305569c28df --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/uncaught.cy.js @@ -0,0 +1,79 @@ +import Bluebird from 'bluebird' + +describe('uncaught errors', { defaultCommandTimeout: 0 }, () => { + it('sync app visit exception', () => { + cy.visit('cypress/fixtures/errors.html?error-on-visit') + cy.get('.trigger-sync-error').click() + }) + + it('sync app navigates to visit exception', () => { + cy.visit('cypress/fixtures/errors.html') + cy.get('.go-to-visit-error').click() + }) + + it('sync app exception', () => { + cy.visit('cypress/fixtures/errors.html') + cy.get('.trigger-sync-error').click() + }) + + it('async app exception', () => { + cy.visit('cypress/fixtures/errors.html') + cy.get('.trigger-async-error').click() + cy.wait(10000) + }) + + it('app unhandled rejection', () => { + cy.visit('cypress/fixtures/errors.html') + cy.get('.trigger-unhandled-rejection').click() + cy.wait(10000) + }) + + // TODO: Cypress.Promise.reject() gets caught by AUT. Can/should + // we handle that somehow? + + it('exception inside uncaught:exception', () => { + cy.on('uncaught:exception', () => { + ({}).bar() + }) + + cy.visit('cypress/fixtures/errors.html') + cy.get('.trigger-sync-error').click() + }) + + it('async spec exception', () => { + setTimeout(() => { + ({}).bar() + }) + + cy.wait(10000) + }) + + // eslint-disable-next-line mocha/handle-done-callback + it('async spec exception with done', (done) => { + setTimeout(() => { + ({}).bar() + }) + }) + + it('spec unhandled rejection', () => { + Promise.reject(new Error('Unhandled promise rejection from the spec')) + + cy.wait(10000) + }) + + // eslint-disable-next-line mocha/handle-done-callback + it('spec unhandled rejection with done', (done) => { + Promise.reject(new Error('Unhandled promise rejection from the spec')) + }) + + it('spec Bluebird unhandled rejection', () => { + Bluebird.reject(new Error('Unhandled promise rejection from the spec')) + + cy.wait(10000) + }) + + // eslint-disable-next-line mocha/handle-done-callback + it('spec Bluebird unhandled rejection with done', (done) => { + Bluebird.reject(new Error('Unhandled promise rejection from the spec')) + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/uncaught_outside_test.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/uncaught_outside_test.cy.js new file mode 100644 index 000000000000..e7ae825a3afc --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/uncaught_outside_test.cy.js @@ -0,0 +1,5 @@ +describe('suite', () => { + it('t1', () => {}) +}) + +throw new Error('error from outside test') diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/uncaught_outside_test_only_suite.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/uncaught_outside_test_only_suite.cy.js new file mode 100644 index 000000000000..04754685a113 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/uncaught_outside_test_only_suite.cy.js @@ -0,0 +1,12 @@ +// eslint-disable-next-line mocha/no-exclusive-tests +describe.only('suite', () => { + it('t1', () => { + + }) + + it('t2', () => { + + }) +}) + +throw new Error('error from outside test with only suite') diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/unexpected.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/unexpected.cy.js new file mode 100644 index 000000000000..5a1bb6417564 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/unexpected.cy.js @@ -0,0 +1,26 @@ +describe('unexpected errors', { defaultCommandTimeout: 0 }, () => { + let originalIsSpecialKeyword + let orignalCyExpect + + beforeEach(() => { + originalIsSpecialKeyword = Cypress.LocalStorage._isSpecialKeyword + orignalCyExpect = cy.expect + }) + + afterEach(() => { + Cypress.LocalStorage._isSpecialKeyword = originalIsSpecialKeyword + cy.expect = orignalCyExpect + }) + + it('Cypress method error', () => { + window.eval('Cypress.LocalStorage._isSpecialKeyword = () => { throw new Error("thrown in Cypress-LocalStorage-_isSpecialKeyword") }') + + Cypress.LocalStorage.clear() + }) + + it('internal cy error', () => { + window.eval('cy.expect = () => { throw new Error("thrown in cy-expect")}') + + cy.wrap({ foo: 'foo' }).should('have.property', 'foo') + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/validation.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/validation.cy.js new file mode 100644 index 000000000000..2a47a635f30c --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/validation.cy.js @@ -0,0 +1,13 @@ +describe('validation errors', { defaultCommandTimeout: 0 }, () => { + it('from cypress', () => { + cy.viewport() + }) + + it('from chai expect', () => { + expect(true).to.be.nope + }) + + it('from chai assert', () => { + assert.deepInclude() + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/visit.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/visit.cy.js new file mode 100644 index 000000000000..01b29a8756a0 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/visit.cy.js @@ -0,0 +1,33 @@ +describe('cy.visit', () => { + it('onBeforeLoad assertion failure', () => { + cy.visit('cypress/fixtures/index.html', { + onBeforeLoad () { + expect('actual').to.equal('expected') + }, + }) + }) + + it('onBeforeLoad exception', () => { + cy.visit('cypress/fixtures/index.html', { + onBeforeLoad () { + ({}).bar() + }, + }) + }) + + it('onLoad assertion failure', () => { + cy.visit('cypress/fixtures/index.html', { + onLoad () { + expect('actual').to.equal('expected') + }, + }) + }) + + it('onLoad exception', () => { + cy.visit('cypress/fixtures/index.html', { + onLoad () { + ({}).bar() + }, + }) + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/within.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/within.cy.js new file mode 100644 index 000000000000..43d93eb69989 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/within.cy.js @@ -0,0 +1,19 @@ +describe('cy.within', { defaultCommandTimeout: 0 }, () => { + it('assertion failure', () => { + cy.get('body').within(() => { + expect('actual').to.equal('expected') + }) + }) + + it('exception', () => { + cy.get('body').within(() => { + ({}).bar() + }) + }) + + it('command failure', () => { + cy.get('body').within(() => { + cy.get('#does-not-exist') + }) + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/wrap.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/wrap.cy.js new file mode 100644 index 000000000000..0bed6d9745c8 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/wrap.cy.js @@ -0,0 +1,19 @@ +describe('cy.wrap', { defaultCommandTimeout: 0 }, () => { + it('assertion failure', () => { + cy.wrap(() => { + expect('actual').to.equal('expected') + }).then((fn) => fn()) + }) + + it('exception', () => { + cy.wrap(() => { + ({}).bar() + }).then((fn) => fn()) + }) + + it('command failure', () => { + cy.wrap(() => { + cy.get('#does-not-exist') + }).then((fn) => fn()) + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/fixtures/errors.html b/system-tests/projects/runner-e2e-specs/cypress/fixtures/errors.html new file mode 100644 index 000000000000..af9eda186391 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/fixtures/errors.html @@ -0,0 +1,69 @@ + + + + Page Title + + +
+

Go to Visit Error

+ + + + +
+

+

+ + + + diff --git a/system-tests/projects/runner-e2e-specs/cypress/fixtures/index.html b/system-tests/projects/runner-e2e-specs/cypress/fixtures/index.html new file mode 100644 index 000000000000..5171c27199e6 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/fixtures/index.html @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/system-tests/projects/runner-e2e-specs/src/throws-error.js b/system-tests/projects/runner-e2e-specs/src/throws-error.js new file mode 100644 index 000000000000..552fe2a33cff --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/src/throws-error.js @@ -0,0 +1,6 @@ +// this file is used by the e2e project various_failures.cy.js +// to test an error being thrown in a file outside the user's project + +module.exports = () => { + throw new Error('An outside error') +} diff --git a/system-tests/projects/runner-e2e-specs/tsconfig.json b/system-tests/projects/runner-e2e-specs/tsconfig.json new file mode 100644 index 000000000000..449e5c53c14b --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/tsconfig.json @@ -0,0 +1,16 @@ +{ + "include": [ + "src/**/*.vue", + "src/**/*.tsx", + "src/**/*.ts", + "cypress/**/*.ts", + "cypress/**/*.tsx", + ], + "compilerOptions": { + "noImplicitThis": true, + "allowJs": true, + "types": [ + "cypress", + ] + } +} From 250a97323c9f2b184a42f44ecb6b1b829396d179 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Tue, 1 Feb 2022 10:45:49 -0600 Subject: [PATCH 02/64] Removing existing integration spec file --- .../runner/cypress/e2e/reporter.errors.cy.js | 782 ------------------ 1 file changed, 782 deletions(-) delete mode 100644 packages/runner/cypress/e2e/reporter.errors.cy.js diff --git a/packages/runner/cypress/e2e/reporter.errors.cy.js b/packages/runner/cypress/e2e/reporter.errors.cy.js deleted file mode 100644 index b92403c550c9..000000000000 --- a/packages/runner/cypress/e2e/reporter.errors.cy.js +++ /dev/null @@ -1,782 +0,0 @@ -import { verify, verifyInternalFailure } from '../support/verify-failures' - -describe('errors ui', () => { - describe('assertion failures', () => { - const file = 'assertions_spec.js' - - verify.it('with expect().', { - file, - column: 25, - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('with assert()', { - file, - column: '(5|12)', // (chrome|firefox) - message: `should be true`, - }) - - verify.it('with assert.()', { - file, - column: 12, - message: `expected 'actual' to equal 'expected'`, - }) - }) - - describe('exception failures', () => { - const file = 'exceptions_spec.js' - - verify.it('in spec file', { - file, - column: 10, - message: 'bar is not a function', - }) - - verify.it('in file outside project', { - file, - message: 'An outside error', - regex: /todos\/throws\-error\.js:5:9/, - codeFrameText: `thrownewError('An outside error')`, - verifyOpenInIde: false, - }) - }) - - describe('hooks', { viewportHeight: 900 }, () => { - const file = 'hooks_spec.js' - - // https://github.com/cypress-io/cypress/issues/8214 - // https://github.com/cypress-io/cypress/issues/8288 - // https://github.com/cypress-io/cypress/issues/8350 - verify.it('errors when a hook is nested in another hook', { - file, - column: '(7|18)', // (chrome|firefox) - codeFrameText: 'beforeEach(()=>', - message: `Cypress detected you registered a(n) beforeEach hook while a test was running`, - }) - }) - - describe('commands', () => { - const file = 'commands_spec.js' - - verify.it('failure', { - file, - column: 8, - message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', - }) - - verify.it('chained failure', { - file, - column: 20, - message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', - }) - }) - - describe('cy.then', () => { - const file = 'then_spec.js' - - verify.it('assertion failure', { - file, - column: 27, - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('exception', { - file, - column: 12, - message: 'bar is not a function', - }) - - verify.it('command failure', { - file, - column: 10, - message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', - }) - }) - - describe('cy.should', () => { - const file = 'should_spec.js' - - verify.it('callback assertion failure', { - file, - column: 27, - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('callback exception', { - file, - column: 12, - message: 'bar is not a function', - }) - - verify.it('standard assertion failure', { - file, - column: 6, - message: 'Timed out retrying after 0ms: expected {} to have property \'foo\'', - }) - - verify.it('after multiple', { - file, - column: 6, - message: 'Timed out retrying after 0ms: expected \'foo\' to equal \'bar\'', - }) - - verify.it('after multiple callbacks exception', { - file, - column: 12, - codeFrameText: '({}).bar()', - message: 'bar is not a function', - }) - - verify.it('after multiple callbacks assertion failure', { - file, - column: 27, - codeFrameText: '.should(()=>', - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('after callback success assertion failure', { - file, - column: 6, - codeFrameText: '.should(\'have.property', - message: `expected {} to have property 'foo'`, - }) - - verify.it('command failure after success', { - file, - column: 8, - message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', - }) - }) - - describe('cy.each', () => { - const file = 'each_spec.js' - - verify.it('assertion failure', { - file, - column: 27, - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('exception', { - file, - column: 12, - message: 'bar is not a function', - }) - - verify.it('command failure', { - file, - column: 10, - message: 'Expected to find element: #does-not-exist, but never found it', - }) - }) - - describe('cy.spread', () => { - const file = 'spread_spec.js' - - verify.it('assertion failure', { - file, - column: 27, - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('exception', { - file, - column: 12, - message: 'bar is not a function', - }) - - verify.it('command failure', { - file, - column: 10, - message: 'Expected to find element: #does-not-exist, but never found it', - }) - }) - - describe('cy.within', () => { - const file = 'within_spec.js' - - verify.it('assertion failure', { - file, - column: 27, - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('exception', { - file, - column: 12, - message: 'bar is not a function', - }) - - verify.it('command failure', { - file, - column: 10, - message: 'Expected to find element: #does-not-exist, but never found it', - }) - }) - - describe('cy.wrap', () => { - const file = 'wrap_spec.js' - - verify.it('assertion failure', { - file, - column: 27, - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('exception', { - file, - column: 12, - message: 'bar is not a function', - }) - - verify.it('command failure', { - file, - column: 10, - message: 'Expected to find element: #does-not-exist, but never found it', - }) - }) - - describe('cy.visit', () => { - const file = 'visit_spec.js' - - verify.it('onBeforeLoad assertion failure', { - file, - column: 29, - codeFrameText: 'onBeforeLoad', - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('onBeforeLoad exception', { - file, - column: 14, - codeFrameText: 'onBeforeLoad', - message: 'bar is not a function', - }) - - verify.it('onLoad assertion failure', { - file, - column: 29, - codeFrameText: 'onLoad', - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('onLoad exception', { - file, - column: 14, - codeFrameText: 'onLoad', - message: 'bar is not a function', - }) - }) - - describe('cy.intercept', () => { - const file = 'intercept_spec.ts' - - verify.it('assertion failure in req callback', { - file, - column: 22, - message: [ - `expected 'a' to equal 'b'`, - ], - notInMessage: [ - 'The following error originated from your spec code', - ], - }) - - verify.it('assertion failure in res callback', { - file, - column: 24, - codeFrameText: '.reply(()=>{', - message: [ - `expected 'b' to equal 'c'`, - ], - notInMessage: [ - 'The following error originated from your spec code', - ], - }) - - verify.it('fails when erroneous response is received while awaiting response', { - file, - column: 6, - // this fails the active test because it's an asynchronous - // response failure from the network - codeFrameText: '.wait(1000)', - message: [ - 'A callback was provided to intercept the upstream response, but a network error occurred while making the request', - ], - notInMessage: [ - 'The following error originated from your spec code', - ], - }) - }) - - describe('cy.route', () => { - const file = 'route_spec.js' - - verify.it('callback assertion failure', { - file, - column: 27, - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('callback exception', { - file, - column: 12, - message: 'bar is not a function', - }) - - verify.it('command failure', { - file, - column: 10, - message: 'Expected to find element: #does-not-exist, but never found it', - }) - - verify.it('onAbort assertion failure', { - file, - column: 29, - codeFrameText: 'onAbort', - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('onAbort exception', { - file, - column: 14, - codeFrameText: 'onAbort', - message: 'bar is not a function', - }) - - verify.it('onRequest assertion failure', { - file, - column: 29, - codeFrameText: 'onRequest', - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('onRequest exception', { - file, - column: 14, - codeFrameText: 'onRequest', - message: 'bar is not a function', - }) - - verify.it('onResponse assertion failure', { - file, - column: 29, - codeFrameText: 'onResponse', - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('onResponse exception', { - file, - column: 14, - codeFrameText: 'onResponse', - message: 'bar is not a function', - }) - }) - - describe('cy.server', () => { - const file = 'server_spec.js' - - verify.it('onAbort assertion failure', { - file, - column: 29, - codeFrameText: 'onAbort', - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('onAbort exception', { - file, - column: 14, - codeFrameText: 'onAbort', - message: 'bar is not a function', - }) - - verify.it('onRequest assertion failure', { - file, - column: 29, - codeFrameText: 'onRequest', - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('onRequest exception', { - file, - column: 14, - codeFrameText: 'onRequest', - message: 'bar is not a function', - }) - - verify.it('onResponse assertion failure', { - file, - column: 29, - codeFrameText: 'onResponse', - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('onResponse exception', { - file, - column: 14, - codeFrameText: 'onResponse', - message: 'bar is not a function', - }) - }) - - describe('cy.readFile', () => { - const file = 'readfile_spec.js' - - verify.it('existence failure', { - onBeforeRun ({ win }) { - win.runnerWs.emit.withArgs('backend:request', 'read:file') - .yields({ error: { code: 'ENOENT' } }) - }, - file, - column: 8, - message: 'failed because the file does not exist', - }) - }) - - describe('validation errors', () => { - const file = 'validation_spec.js' - - verify.it('from cypress', { - file, - column: 8, - message: 'can only accept a string preset or', - stack: ['throwErrBadArgs', 'From Your Spec Code:'], - }) - - verify.it('from chai expect', { - file, - column: '(5|12)', // (chrome|firefox) - message: 'Invalid Chai property: nope', - stack: ['proxyGetter', 'From Your Spec Code:'], - }) - - verify.it('from chai assert', { - file, - column: 12, - message: 'object tested must be an array', - }) - }) - - describe('event handlers', () => { - const file = 'events_spec.js' - - verify.it('event assertion failure', { - file, - column: 27, - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('event exception', { - file, - column: 12, - message: 'bar is not a function', - }) - - verify.it('fail handler assertion failure', { - file, - column: 27, - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('fail handler exception', { - file, - column: 12, - message: 'bar is not a function', - }) - }) - - describe('uncaught errors', () => { - const file = 'uncaught_spec.js' - - verify.it('sync app visit exception', { - file, - uncaught: true, - command: 'visit', - visitUrl: 'http://localhost:3500/fixtures/errors.html?error-on-visit', - originalMessage: 'visit error', - message: [ - 'The following error originated from your application code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - regex: /localhost\:\d+\/fixtures\/errors.html\?error-on-visit:\d+:\d+/, - hasCodeFrame: false, - verifyOpenInIde: false, - }) - - verify.it('sync app navigates to visit exception', { - file, - uncaught: true, - visitUrl: 'http://localhost:3500/fixtures/errors.html', - originalMessage: 'visit error', - message: [ - 'The following error originated from your application code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - regex: /localhost\:\d+\/fixtures\/errors.html\?error-on-visit:\d+:\d+/, - hasCodeFrame: false, - verifyOpenInIde: false, - }) - - verify.it('sync app exception', { - file, - uncaught: true, - command: 'click', - visitUrl: 'http://localhost:3500/fixtures/errors.html', - originalMessage: 'sync error', - message: [ - 'The following error originated from your application code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - regex: /localhost\:\d+\/fixtures\/errors.html:\d+:\d+/, - hasCodeFrame: false, - verifyOpenInIde: false, - }) - - verify.it('async app exception', { - file, - uncaught: true, - visitUrl: 'http://localhost:3500/fixtures/errors.html', - originalMessage: 'async error', - message: [ - 'The following error originated from your application code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - regex: /localhost\:\d+\/fixtures\/errors.html:\d+:\d+/, - hasCodeFrame: false, - verifyOpenInIde: false, - }) - - verify.it('app unhandled rejection', { - file, - uncaught: true, - visitUrl: 'http://localhost:3500/fixtures/errors.html', - originalMessage: 'promise rejection', - message: [ - 'The following error originated from your application code', - 'It was caused by an unhandled promise rejection', - ], - regex: /localhost\:\d+\/fixtures\/errors.html:\d+:\d+/, - hasCodeFrame: false, - verifyOpenInIde: false, - }) - - verify.it('async spec exception', { - file, - uncaught: true, - column: 12, - originalMessage: 'bar is not a function', - message: [ - 'The following error originated from your test code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - }) - - verify.it('async spec exception with done', { - file, - uncaught: true, - column: 12, - originalMessage: 'bar is not a function', - message: [ - 'The following error originated from your test code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - }) - - verify.it('spec unhandled rejection', { - file, - uncaught: true, - column: 20, - originalMessage: 'Unhandled promise rejection from the spec', - message: [ - 'The following error originated from your test code', - 'It was caused by an unhandled promise rejection', - ], - }) - - verify.it('spec unhandled rejection with done', { - file, - uncaught: true, - column: 20, - originalMessage: 'Unhandled promise rejection from the spec', - message: [ - 'The following error originated from your test code', - 'It was caused by an unhandled promise rejection', - ], - }) - - verify.it('spec Bluebird unhandled rejection', { - file, - uncaught: true, - column: 21, - originalMessage: 'Unhandled promise rejection from the spec', - message: [ - 'The following error originated from your test code', - 'It was caused by an unhandled promise rejection', - ], - }) - - verify.it('spec Bluebird unhandled rejection with done', { - file, - uncaught: true, - column: 21, - originalMessage: 'Unhandled promise rejection from the spec', - message: [ - 'The following error originated from your test code', - 'It was caused by an unhandled promise rejection', - ], - }) - - verify.it('exception inside uncaught:exception', { - file, - uncaught: true, - uncaughtMessage: 'sync error', - visitUrl: 'http://localhost:3500/fixtures/errors.html', - column: 12, - originalMessage: 'bar is not a function', - message: [ - 'The following error originated from your test code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - }) - - // NOTE: the following 2 test don't have uncaught: true because we don't - // display command logs if there are only events and not true commands - // and uncaught: true causes the verification to look for the error - // event command log - verify.it('spec exception outside test', { - file: 'uncaught_outside_test_spec.js', - column: 7, - message: [ - 'The following error originated from your test code', - 'error from outside test', - 'Cypress could not associate this error to any specific test', - ], - codeFrameText: `thrownewError('error from outside test')`, - }) - - verify.it('spec exception outside test with only suite', { - file: 'uncaught_outside_test_only_suite_spec.js', - column: 7, - message: [ - 'error from outside test with only suite', - 'The following error originated from your test code', - 'Cypress could not associate this error to any specific test', - ], - codeFrameText: `thrownewError('error from outside test with only suite')`, - }) - }) - - describe('custom commands', () => { - const file = 'custom_commands_spec.js' - - verify.it('assertion failure', { - file, - column: 23, - message: `expected 'actual' to equal 'expected'`, - codeFrameText: `add('failAssertion'`, - }) - - verify.it('exception', { - file, - column: 8, - message: 'bar is not a function', - codeFrameText: `add('failException'`, - }) - - verify.it('command failure', { - file, - column: 6, - message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', - codeFrameText: `add('failCommand'`, - }) - }) - - describe('typescript', () => { - const file = 'typescript_spec.ts' - - verify.it('assertion failure', { - file, - column: 25, - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('exception', { - file, - column: 10, - message: 'bar is not a function', - }) - - verify.it('command failure', { - file, - column: 8, - message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', - }) - }) - - describe('docs url', () => { - after(() => { - window.top.__cySkipValidateConfig = false - }) - - const file = 'docs_url_spec.js' - const docsUrl = 'https://on.cypress.io/viewport' - - window.top.__cySkipValidateConfig = true - verify.it('displays as button in interactive mode', { retries: 1 }, { - file, - verifyFn () { - cy - .get('.runnable-err-message') - .should('not.contain', docsUrl) - .contains('Learn more') - .should('have.attr', 'href', docsUrl) - }, - }) - - verify.it('is text in error message in run mode', { - file, - verifyFn () { - cy - .get('.runnable-err-message') - .should('contain', docsUrl) - .contains('Learn more') - .should('not.exist') - }, - }) - }) - - // cases where there is a bug in Cypress and we should show cypress internals - // instead of the invocation stack. we test this by monkey-patching internal - // methods to make them throw an error - describe('unexpected errors', () => { - const file = 'unexpected_spec.js' - - // FIXME: the eval doesn't seem to take effect and overwrite the method - // so it ends up not failing properly - verify.it.skip('Cypress method error', { - file, - verifyFn: verifyInternalFailure, - method: 'Cypress.LocalStorage._isSpecialKeyword', - }) - - verify.it('internal cy error', { - file, - verifyFn: verifyInternalFailure, - method: 'cy.expect', - }) - }) -}) From 4e89fef713f2a456f8d9f79b151560c14f0c7dfc Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Tue, 1 Feb 2022 11:44:22 -0600 Subject: [PATCH 03/64] Removing existing runner fixtures for error specs --- .../fixtures/errors/assertions_spec.js | 15 --- .../cypress/fixtures/errors/commands_spec.js | 11 -- .../fixtures/errors/custom_commands_spec.js | 27 ---- .../cypress/fixtures/errors/docs_url_spec.js | 13 -- .../cypress/fixtures/errors/each_spec.js | 21 ---- .../cypress/fixtures/errors/events_spec.js | 35 ------ .../fixtures/errors/exceptions_spec.js | 13 -- .../cypress/fixtures/errors/hooks_spec.js | 18 --- .../cypress/fixtures/errors/intercept_spec.ts | 116 ------------------ .../fixtures/errors/nested_hooks_spec.js | 17 --- .../cypress/fixtures/errors/readfile_spec.js | 7 -- .../cypress/fixtures/errors/route_spec.js | 85 ------------- .../cypress/fixtures/errors/server_spec.js | 65 ---------- .../runner/cypress/fixtures/errors/setup.js | 29 ----- .../cypress/fixtures/errors/should_spec.js | 58 --------- .../cypress/fixtures/errors/spread_spec.js | 21 ---- .../cypress/fixtures/errors/then_spec.js | 21 ---- .../fixtures/errors/typescript_spec.ts | 21 ---- .../uncaught_outside_test_only_suite_spec.js | 14 --- .../errors/uncaught_outside_test_spec.js | 7 -- .../cypress/fixtures/errors/uncaught_spec.js | 81 ------------ .../fixtures/errors/unexpected_spec.js | 30 ----- .../fixtures/errors/validation_spec.js | 15 --- .../cypress/fixtures/errors/visit_spec.js | 35 ------ .../cypress/fixtures/errors/within_spec.js | 21 ---- .../cypress/fixtures/errors/wrap_spec.js | 21 ---- 26 files changed, 817 deletions(-) delete mode 100644 packages/runner/cypress/fixtures/errors/assertions_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/commands_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/custom_commands_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/docs_url_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/each_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/events_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/exceptions_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/hooks_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/intercept_spec.ts delete mode 100644 packages/runner/cypress/fixtures/errors/nested_hooks_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/readfile_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/route_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/server_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/setup.js delete mode 100644 packages/runner/cypress/fixtures/errors/should_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/spread_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/then_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/typescript_spec.ts delete mode 100644 packages/runner/cypress/fixtures/errors/uncaught_outside_test_only_suite_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/uncaught_outside_test_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/uncaught_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/unexpected_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/validation_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/visit_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/within_spec.js delete mode 100644 packages/runner/cypress/fixtures/errors/wrap_spec.js diff --git a/packages/runner/cypress/fixtures/errors/assertions_spec.js b/packages/runner/cypress/fixtures/errors/assertions_spec.js deleted file mode 100644 index 5651e2c274ab..000000000000 --- a/packages/runner/cypress/fixtures/errors/assertions_spec.js +++ /dev/null @@ -1,15 +0,0 @@ -import './setup' - -describe('assertion failures', () => { - it('with expect().', () => { - expect('actual').to.equal('expected') - }) - - it('with assert()', () => { - assert(false, 'should be true') - }) - - it('with assert.()', () => { - assert.equal('actual', 'expected') - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/commands_spec.js b/packages/runner/cypress/fixtures/errors/commands_spec.js deleted file mode 100644 index 2f509a585350..000000000000 --- a/packages/runner/cypress/fixtures/errors/commands_spec.js +++ /dev/null @@ -1,11 +0,0 @@ -import './setup' - -describe('commands', { defaultCommandTimeout: 0 }, () => { - it('failure', () => { - cy.get('#does-not-exist') - }) - - it('chained failure', () => { - cy.get('body').find('#does-not-exist') - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/custom_commands_spec.js b/packages/runner/cypress/fixtures/errors/custom_commands_spec.js deleted file mode 100644 index 919a23d68f6c..000000000000 --- a/packages/runner/cypress/fixtures/errors/custom_commands_spec.js +++ /dev/null @@ -1,27 +0,0 @@ -import './setup' - -Cypress.Commands.add('failAssertion', () => { - expect('actual').to.equal('expected') -}) - -Cypress.Commands.add('failException', () => { - ({}).bar() -}) - -Cypress.Commands.add('failCommand', () => { - cy.get('#does-not-exist') -}) - -describe('custom commands', { defaultCommandTimeout: 0 }, () => { - it('assertion failure', () => { - cy.failAssertion() - }) - - it('exception', () => { - cy.failException() - }) - - it('command failure', () => { - cy.failCommand() - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/docs_url_spec.js b/packages/runner/cypress/fixtures/errors/docs_url_spec.js deleted file mode 100644 index f5806c950ec7..000000000000 --- a/packages/runner/cypress/fixtures/errors/docs_url_spec.js +++ /dev/null @@ -1,13 +0,0 @@ -import './setup' - -describe('docs url', () => { - it('displays as button in interactive mode', () => { - Cypress.config('isInteractive', true) - cy.viewport() - }) - - it('is text in error message in run mode', () => { - Cypress.config('isInteractive', false) - cy.viewport() - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/each_spec.js b/packages/runner/cypress/fixtures/errors/each_spec.js deleted file mode 100644 index e95785680ddf..000000000000 --- a/packages/runner/cypress/fixtures/errors/each_spec.js +++ /dev/null @@ -1,21 +0,0 @@ -import './setup' - -describe('cy.each', { defaultCommandTimeout: 0 }, () => { - it('assertion failure', () => { - cy.wrap([1]).each(() => { - expect('actual').to.equal('expected') - }) - }) - - it('exception', () => { - cy.wrap([1]).each(() => { - ({}).bar() - }) - }) - - it('command failure', () => { - cy.wrap([1]).each(() => { - cy.get('#does-not-exist') - }) - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/events_spec.js b/packages/runner/cypress/fixtures/errors/events_spec.js deleted file mode 100644 index 855dd08d5516..000000000000 --- a/packages/runner/cypress/fixtures/errors/events_spec.js +++ /dev/null @@ -1,35 +0,0 @@ -import './setup' - -describe('event handlers', { defaultCommandTimeout: 0 }, () => { - it('event assertion failure', () => { - cy.on('window:load', () => { - expect('actual').to.equal('expected') - }) - - cy.visit('http://localhost:1919') - }) - - it('event exception', () => { - cy.on('window:load', () => { - ({}).bar() - }) - - cy.visit('http://localhost:1919') - }) - - it('fail handler assertion failure', () => { - cy.on('fail', () => { - expect('actual').to.equal('expected') - }) - - cy.get('#does-not-exist') - }) - - it('fail handler exception', () => { - cy.on('fail', () => { - ({}).bar() - }) - - cy.get('#does-not-exist') - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/exceptions_spec.js b/packages/runner/cypress/fixtures/errors/exceptions_spec.js deleted file mode 100644 index 605c9a5091f8..000000000000 --- a/packages/runner/cypress/fixtures/errors/exceptions_spec.js +++ /dev/null @@ -1,13 +0,0 @@ -import './setup' - -const outsideError = require('@tooling/system-tests/projects/todos/throws-error') - -describe('exception failures', () => { - it('in spec file', () => { - ({}).bar() - }) - - it('in file outside project', () => { - outsideError() - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/hooks_spec.js b/packages/runner/cypress/fixtures/errors/hooks_spec.js deleted file mode 100644 index 4da8922fee66..000000000000 --- a/packages/runner/cypress/fixtures/errors/hooks_spec.js +++ /dev/null @@ -1,18 +0,0 @@ -const log = () => { - const r = cy.state('runnable') - - assert(true, `${r.type} - ${r.parent.title || 'root'}`) -} - -describe('nested_hooks', () => { - describe('nested beforeEach', () => { - before(() => { - log() - beforeEach(() => { - log() - }) - }) - - it('test', log) - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/intercept_spec.ts b/packages/runner/cypress/fixtures/errors/intercept_spec.ts deleted file mode 100644 index 16a3deddd48c..000000000000 --- a/packages/runner/cypress/fixtures/errors/intercept_spec.ts +++ /dev/null @@ -1,116 +0,0 @@ -import './setup' - -describe('cy.intercept', () => { - const { $ } = Cypress - - const emitProxyLog = () => Cypress.emit('request:event', 'incoming:request', { - requestId: 1, - method: 'GET', - url: '', - headers: {}, - resourceType: 'other', - originalResourceType: 'other', - }) - - it('assertion failure in req callback', () => { - cy.intercept('/json-content-type', () => { - expect('a').to.eq('b') - }) - .then(() => { - emitProxyLog() - Cypress.emit('net:stubbing:event', 'before:request', { - browserRequestId: 1, - eventId: '1', - subscription: { - // @ts-ignore - routeId: Object.keys(Cypress.state('routes'))[0], - await: true, - }, - data: { - url: '', - }, - }) - }) - .wait(1000) // ensure the failure happens before test ends - }) - - it('assertion failure in res callback', () => { - cy.intercept('/json-content-type', (req) => { - req.reply(() => { - expect('b').to.eq('c') - }) - }) - .then(() => { - emitProxyLog() - Cypress.emit('net:stubbing:event', 'before:request', { - browserRequestId: 1, - eventId: '1', - requestId: '1', - subscription: { - // @ts-ignore - routeId: Object.keys(Cypress.state('routes'))[0], - await: true, - }, - data: { - url: '', - }, - }) - - Cypress.emit('net:stubbing:event', 'before:response', { - eventId: '1', - requestId: '1', - subscription: { - // @ts-ignore - id: Object.values(Cypress.state('routes'))[0].requests['1'].subscriptions[0].subscription.id, - // @ts-ignore - routeId: Object.keys(Cypress.state('routes'))[0], - await: true, - }, - data: { - url: '', - }, - }) - }) - .wait(1000) // ensure the failure happens before test ends - }) - - it('fails when erroneous response is received while awaiting response', () => { - cy.intercept('/fake', (req) => { - req.reply(() => { - throw new Error('this should not be reached') - }) - }) - .then(() => { - emitProxyLog() - Cypress.emit('net:stubbing:event', 'before:request', { - browserRequestId: 1, - eventId: '1', - requestId: '1', - subscription: { - // @ts-ignore - routeId: Object.keys(Cypress.state('routes'))[0], - await: true, - }, - data: { - url: '' - }, - }) - - Cypress.emit('net:stubbing:event', 'network:error', { - eventId: '1', - requestId: '1', - subscription: { - // @ts-ignore - routeId: Object.keys(Cypress.state('routes'))[0], - }, - data: { - error: { - name: 'ResponseError', - message: 'it errored', - }, - }, - }) - }) - .wait(1000) // ensure the failure happens before test ends - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/nested_hooks_spec.js b/packages/runner/cypress/fixtures/errors/nested_hooks_spec.js deleted file mode 100644 index ec847fde4d96..000000000000 --- a/packages/runner/cypress/fixtures/errors/nested_hooks_spec.js +++ /dev/null @@ -1,17 +0,0 @@ -/// -const log = function(){ - const r = cy.state('runnable') - assert(true, `${r.type} - ${r.parent.title || 'root'}`) -} - -describe('nested_hooks', ()=>{ - describe('nested beforeEach', ()=>{ - before(() => { - log() - beforeEach(() => { - log() - }) - }) - it('test', log) - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/readfile_spec.js b/packages/runner/cypress/fixtures/errors/readfile_spec.js deleted file mode 100644 index 141e89113e3c..000000000000 --- a/packages/runner/cypress/fixtures/errors/readfile_spec.js +++ /dev/null @@ -1,7 +0,0 @@ -import './setup' - -describe('cy.readFile', () => { - it('existence failure', () => { - cy.readFile('does-not-exist', { timeout: 0 }) - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/route_spec.js b/packages/runner/cypress/fixtures/errors/route_spec.js deleted file mode 100644 index 889e2be067d2..000000000000 --- a/packages/runner/cypress/fixtures/errors/route_spec.js +++ /dev/null @@ -1,85 +0,0 @@ -import { abortXhr, sendXhr } from './setup' - -describe('cy.route', { defaultCommandTimeout: 0 }, () => { - it('callback assertion failure', () => { - cy.server().route(() => { - expect('actual').to.equal('expected') - }) - }) - - it('callback exception', () => { - cy.server().route(() => { - ({}).bar() - }) - }) - - it('command failure', () => { - cy.server().route(() => { - cy.get('#does-not-exist') - - return '/foo' - }) - }) - - it('onAbort assertion failure', () => { - cy.server().route({ - url: '/foo', - onAbort () { - expect('actual').to.equal('expected') - }, - }) - .window().then(abortXhr('/foo')) - }) - - it('onAbort exception', () => { - cy.server().route({ - url: '/foo', - onAbort () { - ({}).bar() - }, - }) - .window().then(abortXhr('/foo')) - }) - - it('onRequest assertion failure', () => { - cy.server().route({ - url: '/foo', - onRequest () { - expect('actual').to.equal('expected') - }, - }) - .window().then(sendXhr('/foo')) - }) - - it('onRequest exception', () => { - cy.server().route({ - url: '/foo', - onRequest () { - ({}).bar() - }, - }) - .window().then(sendXhr('/foo')) - }) - - it('onResponse assertion failure', () => { - cy.server().route({ - url: '/json-content-type', - onResponse () { - expect('actual').to.equal('expected') - }, - }) - .window().then(sendXhr('/json-content-type')) - .wait(10000) - }) - - it('onResponse exception', () => { - cy.server().route({ - url: '/json-content-type', - onResponse () { - ({}).bar() - }, - }) - .window().then(sendXhr('/json-content-type')) - .wait(10000) - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/server_spec.js b/packages/runner/cypress/fixtures/errors/server_spec.js deleted file mode 100644 index 5f6677520e9f..000000000000 --- a/packages/runner/cypress/fixtures/errors/server_spec.js +++ /dev/null @@ -1,65 +0,0 @@ -import { abortXhr, sendXhr } from './setup' - -describe('cy.server', { defaultCommandTimeout: 0 }, () => { - it('onAbort assertion failure', () => { - cy.server({ - onAbort () { - expect('actual').to.equal('expected') - }, - }) - .route('/foo') - .window().then(abortXhr('/foo')) - }) - - it('onAbort exception', () => { - cy.server({ - onAbort () { - ({}).bar() - }, - }) - .route('/foo') - .window().then(abortXhr('/foo')) - }) - - it('onRequest assertion failure', () => { - cy.server({ - onRequest () { - expect('actual').to.equal('expected') - }, - }) - .route('/foo') - .window().then(sendXhr('/foo')) - }) - - it('onRequest exception', () => { - cy.server({ - onRequest () { - ({}).bar() - }, - }) - .route('/foo') - .window().then(sendXhr('/foo')) - }) - - it('onResponse assertion failure', () => { - cy.server({ - onResponse () { - expect('actual').to.equal('expected') - }, - }) - .route('/json-content-type') - .window().then(sendXhr('/json-content-type')) - .wait(10000) - }) - - it('onResponse exception', () => { - cy.server({ - onResponse () { - ({}).bar() - }, - }) - .route('/json-content-type') - .window().then(sendXhr('/json-content-type')) - .wait(10000) - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/setup.js b/packages/runner/cypress/fixtures/errors/setup.js deleted file mode 100644 index f6d5fb0b83bc..000000000000 --- a/packages/runner/cypress/fixtures/errors/setup.js +++ /dev/null @@ -1,29 +0,0 @@ -const testToRun = window.testToRun -const originalIt = window.it - -window.it = (title, ...args) => { - const itFn = title === testToRun ? originalIt : () => {} - - return itFn(title, ...args) -} - -window.it.only = () => { - throw new Error('Instead of putting .only in the spec-under-test, put it in the corresponding test in the parent spec (reporter.error.spec.js, etc)') -} - -// eslint-disable-next-line -export const sendXhr = (route) => (win) => { - const xhr = new win.XMLHttpRequest() - - xhr.open('GET', route) - xhr.send() - - return xhr -} - -// eslint-disable-next-line -export const abortXhr = (route) => (win) => { - const xhr = sendXhr(route)(win) - - xhr.abort() -} diff --git a/packages/runner/cypress/fixtures/errors/should_spec.js b/packages/runner/cypress/fixtures/errors/should_spec.js deleted file mode 100644 index 5d7158333497..000000000000 --- a/packages/runner/cypress/fixtures/errors/should_spec.js +++ /dev/null @@ -1,58 +0,0 @@ -import './setup' - -describe('cy.should', { defaultCommandTimeout: 0 }, () => { - it('callback assertion failure', () => { - cy.wrap({}).should(() => { - expect('actual').to.equal('expected') - }) - }) - - it('callback exception', () => { - cy.wrap({}).should(() => { - ({}).bar() - }) - }) - - it('standard assertion failure', () => { - cy.wrap({}) - .should('have.property', 'foo') - }) - - it('after multiple', () => { - cy.wrap({ foo: 'foo' }).should('have.property', 'foo') - .should('equal', 'bar') - }) - - it('after multiple callbacks exception', () => { - cy.wrap({ foo: 'foo' }) - .should(() => { - expect(true).to.be.true - }) - .should(() => { - ({}).bar() - }) - }) - - it('after multiple callbacks assertion failure', () => { - cy.wrap({ foo: 'foo' }) - .should(() => { - expect(true).to.be.true - }) - .should(() => { - expect('actual').to.equal('expected') - }) - }) - - it('after callback success assertion failure', () => { - cy.wrap({}) - .should(() => { - expect(true).to.be.true - }) - .should('have.property', 'foo') - }) - - it('command failure after success', () => { - cy.wrap({ foo: 'foo' }).should('have.property', 'foo') - cy.get('#does-not-exist') - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/spread_spec.js b/packages/runner/cypress/fixtures/errors/spread_spec.js deleted file mode 100644 index b6dd1a51bf1c..000000000000 --- a/packages/runner/cypress/fixtures/errors/spread_spec.js +++ /dev/null @@ -1,21 +0,0 @@ -import './setup' - -describe('cy.spread', { defaultCommandTimeout: 0 }, () => { - it('assertion failure', () => { - cy.wrap([1, 2, 3]).spread(() => { - expect('actual').to.equal('expected') - }) - }) - - it('exception', () => { - cy.wrap([1, 2, 3]).spread(() => { - ({}).bar() - }) - }) - - it('command failure', () => { - cy.wrap([1, 2, 3]).spread(() => { - cy.get('#does-not-exist') - }) - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/then_spec.js b/packages/runner/cypress/fixtures/errors/then_spec.js deleted file mode 100644 index 00c45d4bb431..000000000000 --- a/packages/runner/cypress/fixtures/errors/then_spec.js +++ /dev/null @@ -1,21 +0,0 @@ -import './setup' - -describe('cy.then', { defaultCommandTimeout: 0 }, () => { - it('assertion failure', () => { - cy.wrap({}).then(() => { - expect('actual').to.equal('expected') - }) - }) - - it('exception', () => { - cy.wrap({}).then(() => { - ({}).bar() - }) - }) - - it('command failure', () => { - cy.wrap({}).then(() => { - cy.get('#does-not-exist') - }) - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/typescript_spec.ts b/packages/runner/cypress/fixtures/errors/typescript_spec.ts deleted file mode 100644 index 285c310e6404..000000000000 --- a/packages/runner/cypress/fixtures/errors/typescript_spec.ts +++ /dev/null @@ -1,21 +0,0 @@ -import './setup' - -// simple example of typescript types -type Foo = { - something: string -} - -describe('typescript', { defaultCommandTimeout: 0 }, () => { - it('assertion failure', () => { - expect('actual').to.equal('expected') - }) - - it('exception', () => { - // @ts-ignore - ({}).bar() - }) - - it('command failure', () => { - cy.get('#does-not-exist') - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/uncaught_outside_test_only_suite_spec.js b/packages/runner/cypress/fixtures/errors/uncaught_outside_test_only_suite_spec.js deleted file mode 100644 index 7171f5c5e703..000000000000 --- a/packages/runner/cypress/fixtures/errors/uncaught_outside_test_only_suite_spec.js +++ /dev/null @@ -1,14 +0,0 @@ -import './setup' - -// eslint-disable-next-line mocha/no-exclusive-tests -describe.only('suite', () => { - it('t1', () => { - - }) - - it('t2', () => { - - }) -}) - -throw new Error('error from outside test with only suite') diff --git a/packages/runner/cypress/fixtures/errors/uncaught_outside_test_spec.js b/packages/runner/cypress/fixtures/errors/uncaught_outside_test_spec.js deleted file mode 100644 index 72788393cf60..000000000000 --- a/packages/runner/cypress/fixtures/errors/uncaught_outside_test_spec.js +++ /dev/null @@ -1,7 +0,0 @@ -import './setup' - -describe('suite', () => { - it('t1', () => {}) -}) - -throw new Error('error from outside test') diff --git a/packages/runner/cypress/fixtures/errors/uncaught_spec.js b/packages/runner/cypress/fixtures/errors/uncaught_spec.js deleted file mode 100644 index 931f12b96462..000000000000 --- a/packages/runner/cypress/fixtures/errors/uncaught_spec.js +++ /dev/null @@ -1,81 +0,0 @@ -import Bluebird from 'bluebird' - -import './setup' - -describe('uncaught errors', { defaultCommandTimeout: 0 }, () => { - it('sync app visit exception', () => { - cy.visit('/index.html') - cy.get('.trigger-sync-error').click() - }) - - it('sync app navigates to visit exception', () => { - cy.visit('/index.html') - cy.get('.go-to-visit-error').click() - }) - - it('sync app exception', () => { - cy.visit('/index.html') - cy.get('.trigger-sync-error').click() - }) - - it('async app exception', () => { - cy.visit('/index.html') - cy.get('.trigger-async-error').click() - cy.wait(10000) - }) - - it('app unhandled rejection', () => { - cy.visit('/index.html') - cy.get('.trigger-unhandled-rejection').click() - cy.wait(10000) - }) - - it('async spec exception', () => { - setTimeout(() => { - ({}).bar() - }) - - cy.wait(10000) - }) - - // eslint-disable-next-line mocha/handle-done-callback - it('async spec exception with done', (done) => { - setTimeout(() => { - ({}).bar() - }) - }) - - it('spec unhandled rejection', () => { - Promise.reject(new Error('Unhandled promise rejection from the spec')) - - cy.wait(10000) - }) - - // eslint-disable-next-line mocha/handle-done-callback - it('spec unhandled rejection with done', (done) => { - Promise.reject(new Error('Unhandled promise rejection from the spec')) - }) - - it('spec Bluebird unhandled rejection', () => { - Bluebird.reject(new Error('Unhandled promise rejection from the spec')) - - cy.wait(10000) - }) - - // eslint-disable-next-line mocha/handle-done-callback - it('spec Bluebird unhandled rejection with done', (done) => { - Bluebird.reject(new Error('Unhandled promise rejection from the spec')) - }) - - // TODO: Cypress.Promise.reject() gets caught by AUT. Can/should - // we handle that somehow? - - it('exception inside uncaught:exception', () => { - cy.on('uncaught:exception', () => { - ({}).bar() - }) - - cy.visit('/index.html') - cy.get('.trigger-sync-error').click() - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/unexpected_spec.js b/packages/runner/cypress/fixtures/errors/unexpected_spec.js deleted file mode 100644 index e77a3cbef6a7..000000000000 --- a/packages/runner/cypress/fixtures/errors/unexpected_spec.js +++ /dev/null @@ -1,30 +0,0 @@ -import './setup' - -describe('unexpected errors', { defaultCommandTimeout: 0 }, () => { - let originalIsSpecialKeyword - let orignalCyExpect - - beforeEach(() => { - originalIsSpecialKeyword = Cypress.LocalStorage._isSpecialKeyword - orignalCyExpect = cy.expect - }) - - afterEach(() => { - Cypress.LocalStorage._isSpecialKeyword = originalIsSpecialKeyword - cy.expect = orignalCyExpect - }) - - it('Cypress method error', () => { - cy.window().then((win) => { - win.localStorage.foo = 'foo' - window.autWindow.eval(`Cypress.LocalStorage._isSpecialKeyword = () => { debugger; throw new Error('thrown in Cypress-LocalStorage-_isSpecialKeyword') }`) - Cypress.LocalStorage.clear() - }) - }) - - it('internal cy error', () => { - window.autWindow.eval(`cy.expect = () => { throw new Error('thrown in cy-expect') }`) - - cy.wrap({ foo: 'foo' }).should('have.property', 'foo') - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/validation_spec.js b/packages/runner/cypress/fixtures/errors/validation_spec.js deleted file mode 100644 index 21d2e48ad3bc..000000000000 --- a/packages/runner/cypress/fixtures/errors/validation_spec.js +++ /dev/null @@ -1,15 +0,0 @@ -import './setup' - -describe('validation errors', { defaultCommandTimeout: 0 }, () => { - it('from cypress', () => { - cy.viewport() - }) - - it('from chai expect', () => { - expect(true).to.be.nope - }) - - it('from chai assert', () => { - assert.deepInclude() - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/visit_spec.js b/packages/runner/cypress/fixtures/errors/visit_spec.js deleted file mode 100644 index f2b727082670..000000000000 --- a/packages/runner/cypress/fixtures/errors/visit_spec.js +++ /dev/null @@ -1,35 +0,0 @@ -import './setup' - -describe('cy.visit', () => { - it('onBeforeLoad assertion failure', () => { - cy.visit('/index.html', { - onBeforeLoad () { - expect('actual').to.equal('expected') - }, - }) - }) - - it('onBeforeLoad exception', () => { - cy.visit('/index.html', { - onBeforeLoad () { - ({}).bar() - }, - }) - }) - - it('onLoad assertion failure', () => { - cy.visit('/index.html', { - onLoad () { - expect('actual').to.equal('expected') - }, - }) - }) - - it('onLoad exception', () => { - cy.visit('/index.html', { - onLoad () { - ({}).bar() - }, - }) - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/within_spec.js b/packages/runner/cypress/fixtures/errors/within_spec.js deleted file mode 100644 index d7ca3f342351..000000000000 --- a/packages/runner/cypress/fixtures/errors/within_spec.js +++ /dev/null @@ -1,21 +0,0 @@ -import './setup' - -describe('cy.within', { defaultCommandTimeout: 0 }, () => { - it('assertion failure', () => { - cy.get('body').within(() => { - expect('actual').to.equal('expected') - }) - }) - - it('exception', () => { - cy.get('body').within(() => { - ({}).bar() - }) - }) - - it('command failure', () => { - cy.get('body').within(() => { - cy.get('#does-not-exist') - }) - }) -}) diff --git a/packages/runner/cypress/fixtures/errors/wrap_spec.js b/packages/runner/cypress/fixtures/errors/wrap_spec.js deleted file mode 100644 index f566fc572c27..000000000000 --- a/packages/runner/cypress/fixtures/errors/wrap_spec.js +++ /dev/null @@ -1,21 +0,0 @@ -import './setup' - -describe('cy.wrap', { defaultCommandTimeout: 0 }, () => { - it('assertion failure', () => { - cy.wrap(() => { - expect('actual').to.equal('expected') - }).then((fn) => fn()) - }) - - it('exception', () => { - cy.wrap(() => { - ({}).bar() - }).then((fn) => fn()) - }) - - it('command failure', () => { - cy.wrap(() => { - cy.get('#does-not-exist') - }).then((fn) => fn()) - }) -}) From 5a3a2fba678ba22524a99b6c27735c07bd90209b Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Tue, 1 Feb 2022 17:04:09 -0600 Subject: [PATCH 04/64] Fixing ts lint --- packages/app/cypress/e2e/runner/reporter.errors.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts index 636d31423afb..43546d4100a5 100644 --- a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts +++ b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts @@ -5,7 +5,7 @@ const setup = ({ fileName, mockPreferredEditor, onLoadStatsMessage }) => { cy.openProject('runner-e2e-specs') if (mockPreferredEditor) { - // set preferred editor to bypass IDE selection dialog + // set preferred editor to bypass IDE selection dialog cy.withCtx((ctx) => { ctx.coreData.localSettings.availableEditors = [ ...ctx.coreData.localSettings.availableEditors, @@ -975,6 +975,7 @@ describe('errors ui', { // FIXME: the eval doesn't seem to take effect and overwrite the method // so it ends up not failing properly + // @ts-ignore verify.it.skip('Cypress method error', { file, verifyFn: verifyInternalFailure, From ccd5135f17abd6c56f949c78df378d26e9bc1f76 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Tue, 1 Feb 2022 23:14:42 -0600 Subject: [PATCH 05/64] Testing with less tests kept in memory --- packages/app/cypress/e2e/runner/reporter.errors.cy.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts index 43546d4100a5..e4e25505be90 100644 --- a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts +++ b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts @@ -36,6 +36,7 @@ const setup = ({ fileName, mockPreferredEditor, onLoadStatsMessage }) => { describe('errors ui', { viewportHeight: 768, viewportWidth: 1024, + numTestsKeptInMemory: 1, }, () => { describe('assertion failures', () => { before(() => { @@ -415,7 +416,7 @@ describe('errors ui', { column: 6, // this fails the active test because it's an asynchronous // response failure from the network - codeFrameText: '83|.then(()=>{', + codeFrameText: '81|.then(()=>{', message: [ 'A callback was provided to intercept the upstream response, but a network error occurred while making the request', ], @@ -920,6 +921,7 @@ describe('errors ui', { file, column: 8, message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', + codeFrameText: `.get('#does-not-exist')`, }) }) From 3591fbbcc2fc116cee4ac1ae7f4829988fcb3fae Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Wed, 2 Feb 2022 11:55:35 -0600 Subject: [PATCH 06/64] Testing some cypress-in-cypress settings to improve run-mode performance. --- system-tests/projects/runner-e2e-specs/cypress.config.js | 2 ++ .../projects/runner-e2e-specs/cypress/e2e/errors/readfile.cy.js | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/system-tests/projects/runner-e2e-specs/cypress.config.js b/system-tests/projects/runner-e2e-specs/cypress.config.js index d12d96d7e3e0..9d56f48a9633 100644 --- a/system-tests/projects/runner-e2e-specs/cypress.config.js +++ b/system-tests/projects/runner-e2e-specs/cypress.config.js @@ -1,4 +1,6 @@ module.exports = { + numTestsKeptInMemory: 0, + video: false, e2e: { supportFile: false, }, diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/readfile.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/readfile.cy.js index f682bd528d58..e91de8ac8387 100644 --- a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/readfile.cy.js +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/readfile.cy.js @@ -1,5 +1,5 @@ describe('cy.readFile', () => { it('existence failure', () => { - cy.readFile('does-not-exist', { timeout: 0 }) + cy.readFile('does-not-exist', { timeout: 100 }) }) }) From ce12a8a4884729b7326853f8951ee7dd76421952 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Thu, 3 Feb 2022 10:12:27 -0600 Subject: [PATCH 07/64] Reverting changes to doc_url specs --- .../cypress/e2e/runner/reporter.errors.cy.ts | 47 ++++++++++++------- .../cypress/e2e/errors/docs_url.cy.js | 8 +++- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts index e4e25505be90..cbd758a5c2e7 100644 --- a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts +++ b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts @@ -930,32 +930,45 @@ describe('errors ui', { const docsUrl = 'https://on.cypress.io/viewport' before(() => { + // @ts-ignore + window.top.__cySkipValidateConfig = true + setup({ fileName: file, mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:1', + onLoadStatsMessage: 'Failed:2', }) }) - verify.it('displays doc url', { retries: 1 }, { + after(() => { + // @ts-ignore + window.top.__cySkipValidateConfig = false + }) + + verify.it('displays as link in interactive mode', { retries: 1 }, { + file, + verifyFn () { + cy.contains('.runnable-title', 'displays as link in interactive mode') + .closest('.runnable').within(() => { + cy + .get('.runnable-err-message') + .should('not.contain', docsUrl) + .contains('Learn more') + .should('have.attr', 'href', docsUrl) + }) + }, + }) + + verify.it('is text in error message in run mode', { file, verifyFn () { - cy - .contains('.runnable-title', 'displays doc url') + cy.contains('.runnable-title', 'is text in error message in run mode') .closest('.runnable').within(() => { - if (Cypress.config('isInteractive')) { - // renders href in open mode - cy.get('.runnable-err-message') - .should('not.contain', docsUrl) - .contains('Learn more') - .should('have.attr', 'href', docsUrl) - } else { - // renders link as text in run mode - cy.get('.runnable-err-message') - .should('contain', docsUrl) - .contains('Learn more') - .should('not.exist') - } + cy + .get('.runnable-err-message') + .should('contain', docsUrl) + .contains('Learn more') + .should('not.exist') }) }, }) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/docs_url.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/docs_url.cy.js index eaf93f9cbb26..c312bb57728c 100644 --- a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/docs_url.cy.js +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/docs_url.cy.js @@ -1,5 +1,11 @@ describe('docs url', () => { - it('displays doc url', () => { + it('displays as link in interactive mode', () => { + Cypress.config('isInteractive', true) + cy.viewport() + }) + + it('is text in error message in run mode', () => { + Cypress.config('isInteractive', false) cy.viewport() }) }) From 6ca17ec6d4c8071903f2e31590b9b096de0354ac Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Thu, 3 Feb 2022 11:45:16 -0600 Subject: [PATCH 08/64] PR Updates. Inverting default value for 'open in IDE' validation in an attempt to reduce test time. --- .../cypress/e2e/runner/reporter.errors.cy.ts | 332 ++++++++++-------- .../e2e/runner/support/verify-failures.ts | 2 +- .../cypress/e2e/errors/intercept.cy.ts | 4 +- 3 files changed, 186 insertions(+), 152 deletions(-) diff --git a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts index cbd758a5c2e7..133a74906b5a 100644 --- a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts +++ b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts @@ -52,6 +52,7 @@ describe('errors ui', { hasPreferredIde: true, column: 25, message: `expected 'actual' to equal 'expected'`, + verifyOpenInIde: true, }) verify.it('with assert()', { @@ -59,6 +60,7 @@ describe('errors ui', { hasPreferredIde: true, column: '(5|12)', // (chrome|firefox) message: `should be true`, + verifyOpenInIde: true, }) verify.it('with assert.()', { @@ -66,6 +68,7 @@ describe('errors ui', { hasPreferredIde: true, column: 12, message: `expected 'actual' to equal 'expected'`, + verifyOpenInIde: true, }) }) @@ -84,6 +87,7 @@ describe('errors ui', { column: 25, message: `expected 'actual' to equal 'expected'`, codeFrameText: 'with expect().', + verifyOpenInIde: true, }) }) @@ -107,7 +111,6 @@ describe('errors ui', { message: 'An outside error', regex: /\/throws\-error\.js:5:9/, codeFrameText: `thrownewError('An outside error')`, - verifyOpenInIde: false, }) }) @@ -154,6 +157,36 @@ describe('errors ui', { }) }) + describe('cy.then', () => { + const file = 'then.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:3', + }) + }) + + verify.it('assertion failure', { + file, + column: 27, + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('exception', { + file, + column: 12, + message: 'bar is not a function', + }) + + verify.it('command failure', { + file, + column: 10, + message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', + }) + }) + describe('cy.should', () => { const file = 'should.cy.js' @@ -388,7 +421,7 @@ describe('errors ui', { }) }) - verify.it('assertion failure in req callback', { + verify.it('assertion failure in request callback', { file, column: 22, message: [ @@ -399,7 +432,7 @@ describe('errors ui', { ], }) - verify.it('assertion failure in res callback', { + verify.it('assertion failure in response callback', { file, column: 24, codeFrameText: '.reply(()=>{', @@ -414,9 +447,11 @@ describe('errors ui', { verify.it('fails when erroneous response is received while awaiting response', { file, column: 6, + // TODO: determine why code frame output is different in run/open mode // this fails the active test because it's an asynchronous // response failure from the network - codeFrameText: '81|.then(()=>{', + // codeFrameText: '.wait(1000)', + hasCodeFrame: false, message: [ 'A callback was provided to intercept the upstream response, but a network error occurred while making the request', ], @@ -649,163 +684,162 @@ describe('errors ui', { }) }) - verify.it('sync app visit exception', { - file, - uncaught: true, - command: 'visit', - originalMessage: 'visit error', - message: [ - 'The following error originated from your application code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - regex: /localhost\:\d+\/cypress\/fixtures\/errors.html\?error-on-visit:\d+:\d+/, - hasCodeFrame: false, - verifyOpenInIde: false, - }) + context('sync', () => { + verify.it('sync app visit exception', { + file, + uncaught: true, + command: 'visit', + originalMessage: 'visit error', + message: [ + 'The following error originated from your application code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + regex: /localhost\:\d+\/cypress\/fixtures\/errors.html\?error-on-visit:\d+:\d+/, + hasCodeFrame: false, + }) - verify.it('sync app navigates to visit exception', { - file, - uncaught: true, - originalMessage: 'visit error', - message: [ - 'The following error originated from your application code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - regex: /localhost\:\d+\/cypress\/fixtures\/errors.html\?error-on-visit:\d+:\d+/, - hasCodeFrame: false, - verifyOpenInIde: false, - }) + verify.it('sync app navigates to visit exception', { + file, + uncaught: true, + originalMessage: 'visit error', + message: [ + 'The following error originated from your application code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + regex: /localhost\:\d+\/cypress\/fixtures\/errors.html\?error-on-visit:\d+:\d+/, + hasCodeFrame: false, + }) - verify.it('sync app exception', { - file, - uncaught: true, - command: 'click', - originalMessage: 'sync error', - message: [ - 'The following error originated from your application code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - regex: /localhost\:\d+\/cypress\/fixtures\/errors.html:\d+:\d+/, - hasCodeFrame: false, - verifyOpenInIde: false, - }) + verify.it('sync app exception', { + file, + uncaught: true, + command: 'click', + originalMessage: 'sync error', + message: [ + 'The following error originated from your application code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + regex: /localhost\:\d+\/cypress\/fixtures\/errors.html:\d+:\d+/, + hasCodeFrame: false, + }) - verify.it('async app exception', { - file, - uncaught: true, - originalMessage: 'async error', - message: [ - 'The following error originated from your application code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - regex: /localhost\:\d+\/cypress\/fixtures\/errors.html:\d+:\d+/, - hasCodeFrame: false, - verifyOpenInIde: false, + verify.it('exception inside uncaught:exception', { + file, + uncaught: true, + uncaughtMessage: 'sync error', + column: 12, + originalMessage: 'bar is not a function', + message: [ + 'The following error originated from your test code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + }) }) - verify.it('app unhandled rejection', { - file, - uncaught: true, - originalMessage: 'promise rejection', - message: [ - 'The following error originated from your application code', - 'It was caused by an unhandled promise rejection', - ], - regex: /localhost\:\d+\/cypress\/fixtures\/errors.html:\d+:\d+/, - hasCodeFrame: false, - verifyOpenInIde: false, - }) + context('async', () => { + verify.it('async app exception', { + file, + uncaught: true, + originalMessage: 'async error', + message: [ + 'The following error originated from your application code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + regex: /localhost\:\d+\/cypress\/fixtures\/errors.html:\d+:\d+/, + hasCodeFrame: false, + }) - verify.it('async spec exception', { - file, - uncaught: true, - column: 12, - originalMessage: 'bar is not a function', - message: [ - 'The following error originated from your test code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - }) + verify.it('app unhandled rejection', { + file, + uncaught: true, + originalMessage: 'promise rejection', + message: [ + 'The following error originated from your application code', + 'It was caused by an unhandled promise rejection', + ], + regex: /localhost\:\d+\/cypress\/fixtures\/errors.html:\d+:\d+/, + hasCodeFrame: false, + }) - verify.it('async spec exception with done', { - file, - uncaught: true, - column: 12, - originalMessage: 'bar is not a function', - message: [ - 'The following error originated from your test code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - }) + verify.it('async spec exception', { + file, + uncaught: true, + column: 12, + originalMessage: 'bar is not a function', + message: [ + 'The following error originated from your test code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + }) - verify.it('spec unhandled rejection', { - file, - uncaught: true, - column: 20, - originalMessage: 'Unhandled promise rejection from the spec', - message: [ - 'The following error originated from your test code', - 'It was caused by an unhandled promise rejection', - ], - }) + verify.it('async spec exception with done', { + file, + uncaught: true, + column: 12, + originalMessage: 'bar is not a function', + message: [ + 'The following error originated from your test code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + }) - verify.it('spec unhandled rejection with done', { - file, - uncaught: true, - column: 20, - originalMessage: 'Unhandled promise rejection from the spec', - message: [ - 'The following error originated from your test code', - 'It was caused by an unhandled promise rejection', - ], - }) + verify.it('spec unhandled rejection', { + file, + uncaught: true, + column: 20, + originalMessage: 'Unhandled promise rejection from the spec', + message: [ + 'The following error originated from your test code', + 'It was caused by an unhandled promise rejection', + ], + }) - verify.it('spec Bluebird unhandled rejection', { - file, - uncaught: true, - column: 21, - originalMessage: 'Unhandled promise rejection from the spec', - message: [ - 'The following error originated from your test code', - 'It was caused by an unhandled promise rejection', - ], - }) + verify.it('spec unhandled rejection with done', { + file, + uncaught: true, + column: 20, + originalMessage: 'Unhandled promise rejection from the spec', + message: [ + 'The following error originated from your test code', + 'It was caused by an unhandled promise rejection', + ], + }) - verify.it('spec Bluebird unhandled rejection with done', { - file, - uncaught: true, - column: 21, - originalMessage: 'Unhandled promise rejection from the spec', - message: [ - 'The following error originated from your test code', - 'It was caused by an unhandled promise rejection', - ], - }) + verify.it('spec Bluebird unhandled rejection', { + file, + uncaught: true, + column: 21, + originalMessage: 'Unhandled promise rejection from the spec', + message: [ + 'The following error originated from your test code', + 'It was caused by an unhandled promise rejection', + ], + }) - verify.it('exception inside uncaught:exception', { - file, - uncaught: true, - uncaughtMessage: 'sync error', - column: 12, - originalMessage: 'bar is not a function', - message: [ - 'The following error originated from your test code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], + verify.it('spec Bluebird unhandled rejection with done', { + file, + uncaught: true, + column: 21, + originalMessage: 'Unhandled promise rejection from the spec', + message: [ + 'The following error originated from your test code', + 'It was caused by an unhandled promise rejection', + ], + }) }) }) diff --git a/packages/app/cypress/e2e/runner/support/verify-failures.ts b/packages/app/cypress/e2e/runner/support/verify-failures.ts index 716f8a8aa546..2e6705c0e2d0 100644 --- a/packages/app/cypress/e2e/runner/support/verify-failures.ts +++ b/packages/app/cypress/e2e/runner/support/verify-failures.ts @@ -24,7 +24,7 @@ export const verifyFailure = (options) => { const { specTitle, hasCodeFrame = true, - verifyOpenInIde = true, + verifyOpenInIde, hasPreferredIde, column, originalMessage, diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/intercept.cy.ts b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/intercept.cy.ts index 6f1b964a5e3f..0a01419e0c10 100644 --- a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/intercept.cy.ts +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/intercept.cy.ts @@ -10,7 +10,7 @@ describe('cy.intercept', () => { }) } - it('assertion failure in req callback', () => { + it('assertion failure in request callback', () => { cy.intercept('/json-content-type', () => { expect('a').to.eq('b') }) @@ -32,7 +32,7 @@ describe('cy.intercept', () => { .wait(1000) // ensure the failure happens before test ends }) - it('assertion failure in res callback', () => { + it('assertion failure in response callback', () => { cy.intercept('/json-content-type', (req) => { req.reply(() => { expect('b').to.eq('c') From 231f651bd68ac5f2f62aa940fe285dc0321a99fa Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Thu, 3 Feb 2022 11:50:41 -0600 Subject: [PATCH 09/64] Cleaning up ported code --- .../cypress/e2e/errors/route.cy.js | 22 ++++++++++--------- .../cypress/e2e/errors/server.cy.js | 22 ++++++++++--------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/route.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/route.cy.js index 92c1c1ce9f7e..eebddbcbb741 100644 --- a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/route.cy.js +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/route.cy.js @@ -1,18 +1,20 @@ -// eslint-disable-next-line -export const sendXhr = (route) => (win) => { - const xhr = new win.XMLHttpRequest() +const sendXhr = (route) => { + return (win) => { + const xhr = new win.XMLHttpRequest() - xhr.open('GET', route) - xhr.send() + xhr.open('GET', route) + xhr.send() - return xhr + return xhr + } } -// eslint-disable-next-line -export const abortXhr = (route) => (win) => { - const xhr = sendXhr(route)(win) +const abortXhr = (route) => { + return (win) => { + const xhr = sendXhr(route)(win) - xhr.abort() + xhr.abort() + } } describe('cy.route', { defaultCommandTimeout: 0 }, () => { diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/server.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/server.cy.js index cf7c7e8aec28..ca3f98daeaba 100644 --- a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/server.cy.js +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/server.cy.js @@ -1,18 +1,20 @@ -// eslint-disable-next-line -export const sendXhr = (route) => (win) => { - const xhr = new win.XMLHttpRequest() +const sendXhr = (route) => { + return (win) => { + const xhr = new win.XMLHttpRequest() - xhr.open('GET', route) - xhr.send() + xhr.open('GET', route) + xhr.send() - return xhr + return xhr + } } -// eslint-disable-next-line -export const abortXhr = (route) => (win) => { - const xhr = sendXhr(route)(win) +const abortXhr = (route) => { + return (win) => { + const xhr = sendXhr(route)(win) - xhr.abort() + xhr.abort() + } } describe('cy.server', { defaultCommandTimeout: 0 }, () => { From ebe76d06013f1c1be24a84de4496c72f7e2c1405 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Thu, 3 Feb 2022 11:53:19 -0600 Subject: [PATCH 10/64] Update system-tests/projects/runner-e2e-specs/cypress/fixtures/index.html Co-authored-by: Emily Rohrbough --- .../projects/runner-e2e-specs/cypress/fixtures/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system-tests/projects/runner-e2e-specs/cypress/fixtures/index.html b/system-tests/projects/runner-e2e-specs/cypress/fixtures/index.html index 5171c27199e6..aa94f61ee04e 100644 --- a/system-tests/projects/runner-e2e-specs/cypress/fixtures/index.html +++ b/system-tests/projects/runner-e2e-specs/cypress/fixtures/index.html @@ -1,3 +1,3 @@ - \ No newline at end of file + From 7369e9d96d12af41672ce5bf5b82a06f8418ec13 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Thu, 3 Feb 2022 12:36:57 -0600 Subject: [PATCH 11/64] Trying build workaround --- packages/example/bin/build.js | 6 ------ packages/example/package.json | 3 +-- yarn.lock | 4 ++-- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/packages/example/bin/build.js b/packages/example/bin/build.js index 73b3b4331484..3873aabd2046 100644 --- a/packages/example/bin/build.js +++ b/packages/example/bin/build.js @@ -12,11 +12,5 @@ shell.cp('-r', join(resolvePkg('cypress-example-kitchensink'), 'app'), '.') shell.rm('-rf', 'cypress') shell.cp('-r', join(resolvePkg('cypress-example-kitchensink'), 'cypress'), '.') -shell.mv(join('cypress', 'integration'), join('cypress', 'e2e')) -shell.exec(` - for f in cypress/e2e/**/*.spec.js; do - mv -- "$f" "\${f%.spec.js}.cy.js" - done -`) shell.exec('node ./bin/convert.js') diff --git a/packages/example/package.json b/packages/example/package.json index 8d7e6a852119..d45f2f28a227 100644 --- a/packages/example/package.json +++ b/packages/example/package.json @@ -28,8 +28,7 @@ "devDependencies": { "chai": "3.5.0", "cross-env": "6.0.3", - "cypress-example-kitchensink": "cypress-io/cypress-example-kitchensink#feat/use-supportFiles", - "gulp": "4.0.2", + "cypress-example-kitchensink": "cypress-io/cypress-example-kitchensink#8389bb6b09c59c2720ceeaadecc8eaa2d4d6551d", "gulp": "4.0.2", "gulp-clean": "0.4.0", "gulp-gh-pages": "0.6.0-6", "gulp-rev-all": "2.0.2", diff --git a/yarn.lock b/yarn.lock index bfdbc81462ee..9e8456011aef 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17157,9 +17157,9 @@ cyclist@^1.0.1: resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9" integrity sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk= -cypress-example-kitchensink@cypress-io/cypress-example-kitchensink#feat/use-supportFiles: +cypress-example-kitchensink@cypress-io/cypress-example-kitchensink#8389bb6b09c59c2720ceeaadecc8eaa2d4d6551d: version "0.0.0-development" - resolved "https://codeload.github.com/cypress-io/cypress-example-kitchensink/tar.gz/03d4884a83fe5687e866f747768485ff40e61911" + resolved "https://codeload.github.com/cypress-io/cypress-example-kitchensink/tar.gz/8389bb6b09c59c2720ceeaadecc8eaa2d4d6551d" dependencies: npm-run-all "^4.1.2" serve "11.3.0" From 71c572d0300b145d67506ad919674c399f0f0378 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Thu, 3 Feb 2022 13:24:07 -0600 Subject: [PATCH 12/64] Bumping up parallelized builds for app integration tests --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index a019323f61f2..9e8222742d74 100644 --- a/circle.yml +++ b/circle.yml @@ -1265,7 +1265,7 @@ jobs: run-app-integration-tests-chrome: <<: *defaults - parallelism: 2 + parallelism: 5 steps: - run-new-ui-tests: browser: chrome From 984d81fac035a8afac99399c3d7fd1950e389bcd Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Thu, 3 Feb 2022 13:55:29 -0600 Subject: [PATCH 13/64] Removing suspicious tests to validate CI build --- .../cypress/e2e/runner/reporter.errors.cy.ts | 142 +++++++++--------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts index 133a74906b5a..a8920a4c2a8c 100644 --- a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts +++ b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts @@ -533,77 +533,77 @@ describe('errors ui', { }) }) - describe('cy.server', () => { - const file = 'server.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:6', - }) - }) - - verify.it('onAbort assertion failure', { - file, - column: 29, - codeFrameText: 'onAbort', - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('onAbort exception', { - file, - column: 14, - codeFrameText: 'onAbort', - message: 'bar is not a function', - }) - - verify.it('onRequest assertion failure', { - file, - column: 29, - codeFrameText: 'onRequest', - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('onRequest exception', { - file, - column: 14, - codeFrameText: 'onRequest', - message: 'bar is not a function', - }) - - verify.it('onResponse assertion failure', { - file, - column: 29, - codeFrameText: 'onResponse', - message: `expected 'actual' to equal 'expected'`, - }) - - verify.it('onResponse exception', { - file, - column: 14, - codeFrameText: 'onResponse', - message: 'bar is not a function', - }) - }) - - describe('cy.readFile', () => { - const file = 'readfile.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:1', - }) - }) - - verify.it('existence failure', { - file, - column: 8, - message: 'failed because the file does not exist', - }) - }) + // describe('cy.server', () => { + // const file = 'server.cy.js' + + // before(() => { + // setup({ + // fileName: file, + // mockPreferredEditor: false, + // onLoadStatsMessage: 'Failed:6', + // }) + // }) + + // verify.it('onAbort assertion failure', { + // file, + // column: 29, + // codeFrameText: 'onAbort', + // message: `expected 'actual' to equal 'expected'`, + // }) + + // verify.it('onAbort exception', { + // file, + // column: 14, + // codeFrameText: 'onAbort', + // message: 'bar is not a function', + // }) + + // verify.it('onRequest assertion failure', { + // file, + // column: 29, + // codeFrameText: 'onRequest', + // message: `expected 'actual' to equal 'expected'`, + // }) + + // verify.it('onRequest exception', { + // file, + // column: 14, + // codeFrameText: 'onRequest', + // message: 'bar is not a function', + // }) + + // verify.it('onResponse assertion failure', { + // file, + // column: 29, + // codeFrameText: 'onResponse', + // message: `expected 'actual' to equal 'expected'`, + // }) + + // verify.it('onResponse exception', { + // file, + // column: 14, + // codeFrameText: 'onResponse', + // message: 'bar is not a function', + // }) + // }) + + // describe('cy.readFile', () => { + // const file = 'readfile.cy.js' + + // before(() => { + // setup({ + // fileName: file, + // mockPreferredEditor: false, + // onLoadStatsMessage: 'Failed:1', + // }) + // }) + + // verify.it('existence failure', { + // file, + // column: 8, + // message: 'failed because the file does not exist', + // }) + // }) describe('validation errors', () => { const file = 'validation.cy.js' From aeb235eda132f893f1dd2fde97ee8011ab55239b Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Thu, 3 Feb 2022 14:13:16 -0600 Subject: [PATCH 14/64] Removing more suspicious tests to test CI --- .../cypress/e2e/runner/reporter.errors.cy.ts | 122 +++++++++--------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts index a8920a4c2a8c..e44de34133cf 100644 --- a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts +++ b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts @@ -461,77 +461,77 @@ describe('errors ui', { }) }) - describe('cy.route', () => { - const file = 'route.cy.js' + // describe('cy.route', () => { + // const file = 'route.cy.js' - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:9', - }) - }) + // before(() => { + // setup({ + // fileName: file, + // mockPreferredEditor: false, + // onLoadStatsMessage: 'Failed:9', + // }) + // }) - verify.it('callback assertion failure', { - file, - column: 27, - message: `expected 'actual' to equal 'expected'`, - }) + // verify.it('callback assertion failure', { + // file, + // column: 27, + // message: `expected 'actual' to equal 'expected'`, + // }) - verify.it('callback exception', { - file, - column: 12, - message: 'bar is not a function', - }) + // verify.it('callback exception', { + // file, + // column: 12, + // message: 'bar is not a function', + // }) - verify.it('command failure', { - file, - column: 10, - message: 'Expected to find element: #does-not-exist, but never found it', - }) + // verify.it('command failure', { + // file, + // column: 10, + // message: 'Expected to find element: #does-not-exist, but never found it', + // }) - verify.it('onAbort assertion failure', { - file, - column: 29, - codeFrameText: 'onAbort', - message: `expected 'actual' to equal 'expected'`, - }) + // verify.it('onAbort assertion failure', { + // file, + // column: 29, + // codeFrameText: 'onAbort', + // message: `expected 'actual' to equal 'expected'`, + // }) - verify.it('onAbort exception', { - file, - column: 14, - codeFrameText: 'onAbort', - message: 'bar is not a function', - }) + // verify.it('onAbort exception', { + // file, + // column: 14, + // codeFrameText: 'onAbort', + // message: 'bar is not a function', + // }) - verify.it('onRequest assertion failure', { - file, - column: 29, - codeFrameText: 'onRequest', - message: `expected 'actual' to equal 'expected'`, - }) + // verify.it('onRequest assertion failure', { + // file, + // column: 29, + // codeFrameText: 'onRequest', + // message: `expected 'actual' to equal 'expected'`, + // }) - verify.it('onRequest exception', { - file, - column: 14, - codeFrameText: 'onRequest', - message: 'bar is not a function', - }) + // verify.it('onRequest exception', { + // file, + // column: 14, + // codeFrameText: 'onRequest', + // message: 'bar is not a function', + // }) - verify.it('onResponse assertion failure', { - file, - column: 29, - codeFrameText: 'onResponse', - message: `expected 'actual' to equal 'expected'`, - }) + // verify.it('onResponse assertion failure', { + // file, + // column: 29, + // codeFrameText: 'onResponse', + // message: `expected 'actual' to equal 'expected'`, + // }) - verify.it('onResponse exception', { - file, - column: 14, - codeFrameText: 'onResponse', - message: 'bar is not a function', - }) - }) + // verify.it('onResponse exception', { + // file, + // column: 14, + // codeFrameText: 'onResponse', + // message: 'bar is not a function', + // }) + // }) // describe('cy.server', () => { // const file = 'server.cy.js' From 532da7bebe8fe5cbf2080250b82a1abb7d9fcf65 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Thu, 3 Feb 2022 15:01:03 -0600 Subject: [PATCH 15/64] Trying to not scaffold every navigation --- .../cypress/e2e/runner/reporter.errors.cy.ts | 293 +++++++++--------- 1 file changed, 148 insertions(+), 145 deletions(-) diff --git a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts index e44de34133cf..5429854d5e75 100644 --- a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts +++ b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts @@ -1,7 +1,6 @@ import { verify, verifyInternalFailure } from './support/verify-failures' -const setup = ({ fileName, mockPreferredEditor, onLoadStatsMessage }) => { - cy.scaffoldProject('runner-e2e-specs') +const setup = ({ fileName, onLoadStatsMessage, mockPreferredEditor }) => { cy.openProject('runner-e2e-specs') if (mockPreferredEditor) { @@ -38,6 +37,10 @@ describe('errors ui', { viewportWidth: 1024, numTestsKeptInMemory: 1, }, () => { + before(() => { + cy.scaffoldProject('runner-e2e-specs') + }) + describe('assertion failures', () => { before(() => { setup({ @@ -461,149 +464,149 @@ describe('errors ui', { }) }) - // describe('cy.route', () => { - // const file = 'route.cy.js' - - // before(() => { - // setup({ - // fileName: file, - // mockPreferredEditor: false, - // onLoadStatsMessage: 'Failed:9', - // }) - // }) - - // verify.it('callback assertion failure', { - // file, - // column: 27, - // message: `expected 'actual' to equal 'expected'`, - // }) - - // verify.it('callback exception', { - // file, - // column: 12, - // message: 'bar is not a function', - // }) - - // verify.it('command failure', { - // file, - // column: 10, - // message: 'Expected to find element: #does-not-exist, but never found it', - // }) - - // verify.it('onAbort assertion failure', { - // file, - // column: 29, - // codeFrameText: 'onAbort', - // message: `expected 'actual' to equal 'expected'`, - // }) - - // verify.it('onAbort exception', { - // file, - // column: 14, - // codeFrameText: 'onAbort', - // message: 'bar is not a function', - // }) - - // verify.it('onRequest assertion failure', { - // file, - // column: 29, - // codeFrameText: 'onRequest', - // message: `expected 'actual' to equal 'expected'`, - // }) - - // verify.it('onRequest exception', { - // file, - // column: 14, - // codeFrameText: 'onRequest', - // message: 'bar is not a function', - // }) - - // verify.it('onResponse assertion failure', { - // file, - // column: 29, - // codeFrameText: 'onResponse', - // message: `expected 'actual' to equal 'expected'`, - // }) - - // verify.it('onResponse exception', { - // file, - // column: 14, - // codeFrameText: 'onResponse', - // message: 'bar is not a function', - // }) - // }) - - // describe('cy.server', () => { - // const file = 'server.cy.js' - - // before(() => { - // setup({ - // fileName: file, - // mockPreferredEditor: false, - // onLoadStatsMessage: 'Failed:6', - // }) - // }) - - // verify.it('onAbort assertion failure', { - // file, - // column: 29, - // codeFrameText: 'onAbort', - // message: `expected 'actual' to equal 'expected'`, - // }) - - // verify.it('onAbort exception', { - // file, - // column: 14, - // codeFrameText: 'onAbort', - // message: 'bar is not a function', - // }) - - // verify.it('onRequest assertion failure', { - // file, - // column: 29, - // codeFrameText: 'onRequest', - // message: `expected 'actual' to equal 'expected'`, - // }) - - // verify.it('onRequest exception', { - // file, - // column: 14, - // codeFrameText: 'onRequest', - // message: 'bar is not a function', - // }) - - // verify.it('onResponse assertion failure', { - // file, - // column: 29, - // codeFrameText: 'onResponse', - // message: `expected 'actual' to equal 'expected'`, - // }) - - // verify.it('onResponse exception', { - // file, - // column: 14, - // codeFrameText: 'onResponse', - // message: 'bar is not a function', - // }) - // }) - - // describe('cy.readFile', () => { - // const file = 'readfile.cy.js' - - // before(() => { - // setup({ - // fileName: file, - // mockPreferredEditor: false, - // onLoadStatsMessage: 'Failed:1', - // }) - // }) - - // verify.it('existence failure', { - // file, - // column: 8, - // message: 'failed because the file does not exist', - // }) - // }) + describe('cy.route', () => { + const file = 'route.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:9', + }) + }) + + verify.it('callback assertion failure', { + file, + column: 27, + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('callback exception', { + file, + column: 12, + message: 'bar is not a function', + }) + + verify.it('command failure', { + file, + column: 10, + message: 'Expected to find element: #does-not-exist, but never found it', + }) + + verify.it('onAbort assertion failure', { + file, + column: 29, + codeFrameText: 'onAbort', + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('onAbort exception', { + file, + column: 14, + codeFrameText: 'onAbort', + message: 'bar is not a function', + }) + + verify.it('onRequest assertion failure', { + file, + column: 29, + codeFrameText: 'onRequest', + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('onRequest exception', { + file, + column: 14, + codeFrameText: 'onRequest', + message: 'bar is not a function', + }) + + verify.it('onResponse assertion failure', { + file, + column: 29, + codeFrameText: 'onResponse', + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('onResponse exception', { + file, + column: 14, + codeFrameText: 'onResponse', + message: 'bar is not a function', + }) + }) + + describe('cy.server', () => { + const file = 'server.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:6', + }) + }) + + verify.it('onAbort assertion failure', { + file, + column: 29, + codeFrameText: 'onAbort', + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('onAbort exception', { + file, + column: 14, + codeFrameText: 'onAbort', + message: 'bar is not a function', + }) + + verify.it('onRequest assertion failure', { + file, + column: 29, + codeFrameText: 'onRequest', + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('onRequest exception', { + file, + column: 14, + codeFrameText: 'onRequest', + message: 'bar is not a function', + }) + + verify.it('onResponse assertion failure', { + file, + column: 29, + codeFrameText: 'onResponse', + message: `expected 'actual' to equal 'expected'`, + }) + + verify.it('onResponse exception', { + file, + column: 14, + codeFrameText: 'onResponse', + message: 'bar is not a function', + }) + }) + + describe('cy.readFile', () => { + const file = 'readfile.cy.js' + + before(() => { + setup({ + fileName: file, + mockPreferredEditor: false, + onLoadStatsMessage: 'Failed:1', + }) + }) + + verify.it('existence failure', { + file, + column: 8, + message: 'failed because the file does not exist', + }) + }) describe('validation errors', () => { const file = 'validation.cy.js' From e2ae1e50a054f10e0db9ed385eed64a7112bcaf0 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Thu, 3 Feb 2022 23:09:37 -0600 Subject: [PATCH 16/64] Bumping resources for run-app-integration job. Updating reporter.errors specs to appropriately reduce AUT refreshes. --- circle.yml | 3 +- .../cypress/e2e/runner/reporter.errors.cy.ts | 907 +++++++----------- .../e2e/runner/support/verify-failures.ts | 21 +- 3 files changed, 363 insertions(+), 568 deletions(-) diff --git a/circle.yml b/circle.yml index 9e8222742d74..2b5305ba9c56 100644 --- a/circle.yml +++ b/circle.yml @@ -1265,7 +1265,8 @@ jobs: run-app-integration-tests-chrome: <<: *defaults - parallelism: 5 + resource_class: medium + parallelism: 4 steps: - run-new-ui-tests: browser: chrome diff --git a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts index 5429854d5e75..3cf7b33da4ba 100644 --- a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts +++ b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts @@ -1,11 +1,13 @@ -import { verify, verifyInternalFailure } from './support/verify-failures' - -const setup = ({ fileName, onLoadStatsMessage, mockPreferredEditor }) => { - cy.openProject('runner-e2e-specs') - - if (mockPreferredEditor) { - // set preferred editor to bypass IDE selection dialog - cy.withCtx((ctx) => { +import { createVerify, verifyInternalFailure } from './support/verify-failures' + +const loadSpec = ({ + fileName, + onLoadStatsMessage, + hasPreferredIde = false, +}) => { + cy.withCtx((ctx, options) => { + if (options.hasPreferredIde) { + // set preferred editor to bypass IDE selection dialog ctx.coreData.localSettings.availableEditors = [ ...ctx.coreData.localSettings.availableEditors, { @@ -16,12 +18,16 @@ const setup = ({ fileName, onLoadStatsMessage, mockPreferredEditor }) => { ] ctx.coreData.localSettings.preferences.preferredEditorBinary = 'test-editor' - }) - } + } + + ctx.coreData.localSettings.preferences.isSpecsListOpen = false + }, { hasPreferredIde }) cy.startAppServer() cy.visitApp() + // directly visiting the spec will sometimes hang, going through + // specs page first to mitigate cy.contains('[data-cy=spec-item]', fileName).click() cy.location().should((location) => { @@ -29,7 +35,10 @@ const setup = ({ fileName, onLoadStatsMessage, mockPreferredEditor }) => { }) // Wait for specs to complete - cy.findByLabelText('Stats').get('.failed', { timeout: 10000 }).should('have.text', onLoadStatsMessage) + cy.findByLabelText('Stats', { timeout: 30000 }) + .get('.failed', { timeout: 10000 }).should('have.text', onLoadStatsMessage) + + return createVerify({ fileName, hasPreferredIde }) } describe('errors ui', { @@ -37,54 +46,44 @@ describe('errors ui', { viewportWidth: 1024, numTestsKeptInMemory: 1, }, () => { - before(() => { + beforeEach(() => { cy.scaffoldProject('runner-e2e-specs') + cy.openProject('runner-e2e-specs') }) - describe('assertion failures', () => { - before(() => { - setup({ - fileName: 'assertions.cy.js', - mockPreferredEditor: true, - onLoadStatsMessage: 'Failed:3', - }) + it('assertion failures', () => { + const verify = loadSpec({ + fileName: 'assertions.cy.js', + hasPreferredIde: true, + onLoadStatsMessage: 'Failed:3', }) - verify.it('with expect().', { - file: 'assertions.cy.js', - hasPreferredIde: true, + verify('with expect().', { column: 25, message: `expected 'actual' to equal 'expected'`, verifyOpenInIde: true, }) - verify.it('with assert()', { - file: 'assertions.cy.js', - hasPreferredIde: true, + verify('with assert()', { column: '(5|12)', // (chrome|firefox) message: `should be true`, verifyOpenInIde: true, }) - verify.it('with assert.()', { - file: 'assertions.cy.js', - hasPreferredIde: true, + verify('with assert.()', { column: 12, message: `expected 'actual' to equal 'expected'`, verifyOpenInIde: true, }) }) - describe('assertion failures - no preferred IDE', () => { - before(() => { - setup({ - fileName: 'assertions.cy.js', - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:3', - }) + it('assertion failures - no preferred IDE', () => { + const verify = loadSpec({ + fileName: 'assertions.cy.js', + onLoadStatsMessage: 'Failed:3', }) - verify.it('with expect().', { + verify('with expect().', { file: 'assertions.cy.js', hasPreferredIde: false, column: 25, @@ -94,338 +93,255 @@ describe('errors ui', { }) }) - describe('exception failures', () => { - before(() => { - setup({ - fileName: 'exceptions.cy.js', - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:2', - }) + it('exception failures', () => { + const verify = loadSpec({ + fileName: 'exceptions.cy.js', + onLoadStatsMessage: 'Failed:2', }) - verify.it('in spec file', { - file: 'exceptions.cy.js', + verify('in spec file', { column: 10, message: 'bar is not a function', }) - verify.it('in file outside project', { - file: 'exceptions.cy.js', + verify('in file outside project', { message: 'An outside error', regex: /\/throws\-error\.js:5:9/, codeFrameText: `thrownewError('An outside error')`, }) }) - describe('hooks', { viewportHeight: 900 }, () => { - before(() => { - setup({ - fileName: 'hooks.cy.js', - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:1', - }) + it('hooks', { viewportHeight: 900 }, () => { + const verify = loadSpec({ + fileName: 'hooks.cy.js', + onLoadStatsMessage: 'Failed:1', }) // https://github.com/cypress-io/cypress/issues/8214 // https://github.com/cypress-io/cypress/issues/8288 // https://github.com/cypress-io/cypress/issues/8350 - verify.it('errors when a hook is nested in another hook', { - file: 'hooks.cy.js', - specTitle: 'test', + verify('test', { column: '(7|18)', // (chrome|firefox) codeFrameText: 'beforeEach(()=>', message: `Cypress detected you registered a(n) beforeEach hook while a test was running`, }) }) - describe('commands', () => { - before(() => { - setup({ - fileName: 'commands.cy.js', - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:2', - }) + it('commands', () => { + const verify = loadSpec({ + fileName: 'commands.cy.js', + onLoadStatsMessage: 'Failed:2', }) - verify.it('failure', { - file: 'commands.cy.js', + verify('failure', { column: 8, message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', }) - verify.it('chained failure', { - file: 'commands.cy.js', + verify('chained failure', { column: 20, message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', }) }) - describe('cy.then', () => { - const file = 'then.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:3', - }) + it('cy.then', () => { + const verify = loadSpec({ + fileName: 'then.cy.js', + onLoadStatsMessage: 'Failed:3', }) - verify.it('assertion failure', { - file, + verify('assertion failure', { column: 27, message: `expected 'actual' to equal 'expected'`, }) - verify.it('exception', { - file, + verify('exception', { column: 12, message: 'bar is not a function', }) - verify.it('command failure', { - file, + verify('command failure', { column: 10, message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', }) }) - describe('cy.should', () => { - const file = 'should.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:8', - }) + it('cy.should', () => { + const verify = loadSpec({ + fileName: 'should.cy.js', + onLoadStatsMessage: 'Failed:8', }) - verify.it('callback assertion failure', { - file, + verify('callback assertion failure', { column: 27, message: `expected 'actual' to equal 'expected'`, }) - verify.it('callback exception', { - file, + verify('callback exception', { column: 12, message: 'bar is not a function', }) - verify.it('standard assertion failure', { - file, + verify('standard assertion failure', { column: 6, message: 'Timed out retrying after 0ms: expected {} to have property \'foo\'', }) - verify.it('after multiple', { - file, + verify('after multiple', { column: 6, message: 'Timed out retrying after 0ms: expected \'foo\' to equal \'bar\'', }) - verify.it('after multiple callbacks exception', { - file, + verify('after multiple callbacks exception', { column: 12, codeFrameText: '({}).bar()', message: 'bar is not a function', }) - verify.it('after multiple callbacks assertion failure', { - file, + verify('after multiple callbacks assertion failure', { column: 27, codeFrameText: '.should(()=>', message: `expected 'actual' to equal 'expected'`, }) - verify.it('after callback success assertion failure', { - file, + verify('after callback success assertion failure', { column: 6, codeFrameText: '.should(\'have.property', message: `expected {} to have property 'foo'`, }) - verify.it('command failure after success', { - file, + verify('command failure after success', { column: 8, message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', }) }) - describe('cy.each', () => { - const file = 'each.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:3', - }) + it('cy.each', () => { + const verify = loadSpec({ + fileName: 'each.cy.js', + onLoadStatsMessage: 'Failed:3', }) - verify.it('assertion failure', { - file, + verify('assertion failure', { column: 27, message: `expected 'actual' to equal 'expected'`, }) - verify.it('exception', { - file, + verify('exception', { column: 12, message: 'bar is not a function', }) - verify.it('command failure', { - file, + verify('command failure', { column: 10, message: 'Expected to find element: #does-not-exist, but never found it', }) }) - describe('cy.spread', () => { - const file = 'spread.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:3', - }) + it('cy.spread', () => { + const verify = loadSpec({ + fileName: 'spread.cy.js', + onLoadStatsMessage: 'Failed:3', }) - verify.it('assertion failure', { - file, + verify('assertion failure', { column: 27, message: `expected 'actual' to equal 'expected'`, }) - verify.it('exception', { - file, + verify('exception', { column: 12, message: 'bar is not a function', }) - verify.it('command failure', { - file, + verify('command failure', { column: 10, message: 'Expected to find element: #does-not-exist, but never found it', }) }) - describe('cy.within', () => { - const file = 'within.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:3', - }) + it('cy.within', () => { + const verify = loadSpec({ + fileName: 'within.cy.js', + onLoadStatsMessage: 'Failed:3', }) - verify.it('assertion failure', { - file, + verify('assertion failure', { column: 27, message: `expected 'actual' to equal 'expected'`, }) - verify.it('exception', { - file, + verify('exception', { column: 12, message: 'bar is not a function', }) - verify.it('command failure', { - file, + verify('command failure', { column: 10, message: 'Expected to find element: #does-not-exist, but never found it', }) }) - describe('cy.wrap', () => { - const file = 'wrap.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:3', - }) + it('cy.wrap', () => { + const verify = loadSpec({ + fileName: 'wrap.cy.js', + onLoadStatsMessage: 'Failed:3', }) - verify.it('assertion failure', { - file, + verify('assertion failure', { column: 27, message: `expected 'actual' to equal 'expected'`, }) - verify.it('exception', { - file, + verify('exception', { column: 12, message: 'bar is not a function', }) - verify.it('command failure', { - file, + verify('command failure', { column: 10, message: 'Expected to find element: #does-not-exist, but never found it', }) }) - describe('cy.visit', () => { - const file = 'visit.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:4', - }) + it('cy.visit', () => { + const verify = loadSpec({ + fileName: 'visit.cy.js', + onLoadStatsMessage: 'Failed:3', }) - verify.it('onBeforeLoad assertion failure', { - file, + verify('onBeforeLoad assertion failure', { column: 29, codeFrameText: 'onBeforeLoad', message: `expected 'actual' to equal 'expected'`, }) - verify.it('onBeforeLoad exception', { - file, + verify('onBeforeLoad exception', { column: 14, codeFrameText: 'onBeforeLoad', message: 'bar is not a function', }) - verify.it('onLoad assertion failure', { - file, + verify('onLoad assertion failure', { column: 29, codeFrameText: 'onLoad', message: `expected 'actual' to equal 'expected'`, }) - verify.it('onLoad exception', { - file, + verify('onLoad exception', { column: 14, codeFrameText: 'onLoad', message: 'bar is not a function', }) }) - describe('cy.intercept', () => { - const file = 'intercept.cy.ts' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:3', - }) + it('cy.intercept', () => { + const verify = loadSpec({ + fileName: 'intercept.cy.ts', + onLoadStatsMessage: 'Failed:3', }) - verify.it('assertion failure in request callback', { - file, + verify('assertion failure in request callback', { column: 22, message: [ `expected 'a' to equal 'b'`, @@ -435,8 +351,7 @@ describe('errors ui', { ], }) - verify.it('assertion failure in response callback', { - file, + verify('assertion failure in response callback', { column: 24, codeFrameText: '.reply(()=>{', message: [ @@ -447,8 +362,7 @@ describe('errors ui', { ], }) - verify.it('fails when erroneous response is received while awaiting response', { - file, + verify('fails when erroneous response is received while awaiting response', { column: 6, // TODO: determine why code frame output is different in run/open mode // this fails the active test because it's an asynchronous @@ -464,407 +378,331 @@ describe('errors ui', { }) }) - describe('cy.route', () => { - const file = 'route.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:9', - }) + it('cy.route', () => { + const verify = loadSpec({ + fileName: 'route.cy.js', + onLoadStatsMessage: 'Failed:9', }) - verify.it('callback assertion failure', { - file, + verify('callback assertion failure', { column: 27, message: `expected 'actual' to equal 'expected'`, }) - verify.it('callback exception', { - file, + verify('callback exception', { column: 12, message: 'bar is not a function', }) - verify.it('command failure', { - file, + verify('command failure', { column: 10, message: 'Expected to find element: #does-not-exist, but never found it', }) - verify.it('onAbort assertion failure', { - file, + verify('onAbort assertion failure', { column: 29, codeFrameText: 'onAbort', message: `expected 'actual' to equal 'expected'`, }) - verify.it('onAbort exception', { - file, + verify('onAbort exception', { column: 14, codeFrameText: 'onAbort', message: 'bar is not a function', }) - verify.it('onRequest assertion failure', { - file, + verify('onRequest assertion failure', { column: 29, codeFrameText: 'onRequest', message: `expected 'actual' to equal 'expected'`, }) - verify.it('onRequest exception', { - file, + verify('onRequest exception', { column: 14, codeFrameText: 'onRequest', message: 'bar is not a function', }) - verify.it('onResponse assertion failure', { - file, + verify('onResponse assertion failure', { column: 29, codeFrameText: 'onResponse', message: `expected 'actual' to equal 'expected'`, }) - verify.it('onResponse exception', { - file, + verify('onResponse exception', { column: 14, codeFrameText: 'onResponse', message: 'bar is not a function', }) }) - describe('cy.server', () => { - const file = 'server.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:6', - }) + it('cy.server', () => { + const verify = loadSpec({ + fileName: 'server.cy.js', + onLoadStatsMessage: 'Failed:6', }) - verify.it('onAbort assertion failure', { - file, + verify('onAbort assertion failure', { column: 29, codeFrameText: 'onAbort', message: `expected 'actual' to equal 'expected'`, }) - verify.it('onAbort exception', { - file, + verify('onAbort exception', { column: 14, codeFrameText: 'onAbort', message: 'bar is not a function', }) - verify.it('onRequest assertion failure', { - file, + verify('onRequest assertion failure', { column: 29, codeFrameText: 'onRequest', message: `expected 'actual' to equal 'expected'`, }) - verify.it('onRequest exception', { - file, + verify('onRequest exception', { column: 14, codeFrameText: 'onRequest', message: 'bar is not a function', }) - verify.it('onResponse assertion failure', { - file, + verify('onResponse assertion failure', { column: 29, codeFrameText: 'onResponse', message: `expected 'actual' to equal 'expected'`, }) - verify.it('onResponse exception', { - file, + verify('onResponse exception', { column: 14, codeFrameText: 'onResponse', message: 'bar is not a function', }) }) - describe('cy.readFile', () => { - const file = 'readfile.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:1', - }) + it('cy.readFile', () => { + const verify = loadSpec({ + fileName: 'readfile.cy.js', + onLoadStatsMessage: 'Failed:1', }) - verify.it('existence failure', { - file, + verify('existence failure', { column: 8, message: 'failed because the file does not exist', }) }) - describe('validation errors', () => { - const file = 'validation.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:3', - }) + it('validation errors', () => { + const verify = loadSpec({ + fileName: 'validation.cy.js', + onLoadStatsMessage: 'Failed:3', }) - verify.it('from cypress', { - file, + verify('from cypress', { column: 8, message: 'can only accept a string preset or', stack: ['throwErrBadArgs', 'From Your Spec Code:'], }) - verify.it('from chai expect', { - file, + verify('from chai expect', { column: '(5|12)', // (chrome|firefox) message: 'Invalid Chai property: nope', stack: ['proxyGetter', 'From Your Spec Code:'], }) - verify.it('from chai assert', { - file, + verify('from chai assert', { column: 12, message: 'object tested must be an array', }) }) - describe('event handlers', () => { - const file = 'events.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:3', - }) + it('event handlers', () => { + const verify = loadSpec({ + fileName: 'events.cy.js', + onLoadStatsMessage: 'Failed:4', }) - verify.it('event assertion failure', { - file, + verify('event assertion failure', { column: 27, message: `expected 'actual' to equal 'expected'`, }) - verify.it('event exception', { - file, + verify('event exception', { column: 12, message: 'bar is not a function', }) - verify.it('fail handler assertion failure', { - file, + verify('fail handler assertion failure', { column: 27, message: `expected 'actual' to equal 'expected'`, }) - verify.it('fail handler exception', { - file, + verify('fail handler exception', { column: 12, message: 'bar is not a function', }) }) - describe('uncaught errors', () => { - const file = 'uncaught.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:11', - }) + it('uncaught errors', () => { + const verify = loadSpec({ + fileName: 'uncaught.cy.js', + onLoadStatsMessage: 'Failed:11', }) - context('sync', () => { - verify.it('sync app visit exception', { - file, - uncaught: true, - command: 'visit', - originalMessage: 'visit error', - message: [ - 'The following error originated from your application code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - regex: /localhost\:\d+\/cypress\/fixtures\/errors.html\?error-on-visit:\d+:\d+/, - hasCodeFrame: false, - }) + verify('sync app visit exception', { + uncaught: true, + command: 'visit', + originalMessage: 'visit error', + message: [ + 'The following error originated from your application code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + regex: /localhost\:\d+\/cypress\/fixtures\/errors.html\?error-on-visit:\d+:\d+/, + hasCodeFrame: false, + }) - verify.it('sync app navigates to visit exception', { - file, - uncaught: true, - originalMessage: 'visit error', - message: [ - 'The following error originated from your application code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - regex: /localhost\:\d+\/cypress\/fixtures\/errors.html\?error-on-visit:\d+:\d+/, - hasCodeFrame: false, - }) + verify('sync app navigates to visit exception', { + uncaught: true, + originalMessage: 'visit error', + message: [ + 'The following error originated from your application code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + regex: /localhost\:\d+\/cypress\/fixtures\/errors.html\?error-on-visit:\d+:\d+/, + hasCodeFrame: false, + }) - verify.it('sync app exception', { - file, - uncaught: true, - command: 'click', - originalMessage: 'sync error', - message: [ - 'The following error originated from your application code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - regex: /localhost\:\d+\/cypress\/fixtures\/errors.html:\d+:\d+/, - hasCodeFrame: false, - }) + verify('sync app exception', { + uncaught: true, + command: 'click', + originalMessage: 'sync error', + message: [ + 'The following error originated from your application code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + regex: /localhost\:\d+\/cypress\/fixtures\/errors.html:\d+:\d+/, + hasCodeFrame: false, + }) - verify.it('exception inside uncaught:exception', { - file, - uncaught: true, - uncaughtMessage: 'sync error', - column: 12, - originalMessage: 'bar is not a function', - message: [ - 'The following error originated from your test code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - }) + verify('exception inside uncaught:exception', { + uncaught: true, + uncaughtMessage: 'sync error', + column: 12, + originalMessage: 'bar is not a function', + message: [ + 'The following error originated from your test code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], }) - context('async', () => { - verify.it('async app exception', { - file, - uncaught: true, - originalMessage: 'async error', - message: [ - 'The following error originated from your application code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - regex: /localhost\:\d+\/cypress\/fixtures\/errors.html:\d+:\d+/, - hasCodeFrame: false, - }) + verify('async app exception', { + uncaught: true, + originalMessage: 'async error', + message: [ + 'The following error originated from your application code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + regex: /localhost\:\d+\/cypress\/fixtures\/errors.html:\d+:\d+/, + hasCodeFrame: false, + }) - verify.it('app unhandled rejection', { - file, - uncaught: true, - originalMessage: 'promise rejection', - message: [ - 'The following error originated from your application code', - 'It was caused by an unhandled promise rejection', - ], - regex: /localhost\:\d+\/cypress\/fixtures\/errors.html:\d+:\d+/, - hasCodeFrame: false, - }) + verify('app unhandled rejection', { + uncaught: true, + originalMessage: 'promise rejection', + message: [ + 'The following error originated from your application code', + 'It was caused by an unhandled promise rejection', + ], + regex: /localhost\:\d+\/cypress\/fixtures\/errors.html:\d+:\d+/, + hasCodeFrame: false, + }) - verify.it('async spec exception', { - file, - uncaught: true, - column: 12, - originalMessage: 'bar is not a function', - message: [ - 'The following error originated from your test code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - }) + verify('async spec exception', { + uncaught: true, + column: 12, + originalMessage: 'bar is not a function', + message: [ + 'The following error originated from your test code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + }) - verify.it('async spec exception with done', { - file, - uncaught: true, - column: 12, - originalMessage: 'bar is not a function', - message: [ - 'The following error originated from your test code', - ], - notInMessage: [ - 'It was caused by an unhandled promise rejection', - ], - }) + verify('async spec exception with done', { + uncaught: true, + column: 12, + originalMessage: 'bar is not a function', + message: [ + 'The following error originated from your test code', + ], + notInMessage: [ + 'It was caused by an unhandled promise rejection', + ], + }) - verify.it('spec unhandled rejection', { - file, - uncaught: true, - column: 20, - originalMessage: 'Unhandled promise rejection from the spec', - message: [ - 'The following error originated from your test code', - 'It was caused by an unhandled promise rejection', - ], - }) + verify('spec unhandled rejection', { + uncaught: true, + column: 20, + originalMessage: 'Unhandled promise rejection from the spec', + message: [ + 'The following error originated from your test code', + 'It was caused by an unhandled promise rejection', + ], + }) - verify.it('spec unhandled rejection with done', { - file, - uncaught: true, - column: 20, - originalMessage: 'Unhandled promise rejection from the spec', - message: [ - 'The following error originated from your test code', - 'It was caused by an unhandled promise rejection', - ], - }) + verify('spec unhandled rejection with done', { + uncaught: true, + column: 20, + originalMessage: 'Unhandled promise rejection from the spec', + message: [ + 'The following error originated from your test code', + 'It was caused by an unhandled promise rejection', + ], + }) - verify.it('spec Bluebird unhandled rejection', { - file, - uncaught: true, - column: 21, - originalMessage: 'Unhandled promise rejection from the spec', - message: [ - 'The following error originated from your test code', - 'It was caused by an unhandled promise rejection', - ], - }) + verify('spec Bluebird unhandled rejection', { + uncaught: true, + column: 21, + originalMessage: 'Unhandled promise rejection from the spec', + message: [ + 'The following error originated from your test code', + 'It was caused by an unhandled promise rejection', + ], + }) - verify.it('spec Bluebird unhandled rejection with done', { - file, - uncaught: true, - column: 21, - originalMessage: 'Unhandled promise rejection from the spec', - message: [ - 'The following error originated from your test code', - 'It was caused by an unhandled promise rejection', - ], - }) + verify('spec Bluebird unhandled rejection with done', { + uncaught: true, + column: 21, + originalMessage: 'Unhandled promise rejection from the spec', + message: [ + 'The following error originated from your test code', + 'It was caused by an unhandled promise rejection', + ], }) }) - describe('uncaught errors: outside test', () => { - const file = 'uncaught_outside_test.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:1', - }) + it('uncaught errors: outside test', () => { + const verify = loadSpec({ + fileName: 'uncaught_outside_test.cy.js', + onLoadStatsMessage: 'Failed:1', }) // NOTE: the following 2 test don't have uncaught: true because we don't // display command logs if there are only events and not true commands // and uncaught: true causes the verification to look for the error // event command log - verify.it('spec exception outside test', { - file, + verify('An uncaught error was detected outside of a test', { column: 7, - specTitle: 'An uncaught error was detected outside of a test', message: [ 'The following error originated from your test code', 'error from outside test', @@ -874,21 +712,14 @@ describe('errors ui', { }) }) - describe('uncaught errors: outside test only suite', () => { - const file = 'uncaught_outside_test_only_suite.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:1', - }) + it('uncaught errors: outside test only suite', () => { + const verify = loadSpec({ + fileName: 'uncaught_outside_test_only_suite.cy.js', + onLoadStatsMessage: 'Failed:1', }) - verify.it('spec exception outside test with only suite', { - file, + verify('An uncaught error was detected outside of a test', { column: 7, - specTitle: 'An uncaught error was detected outside of a test', message: [ 'error from outside test with only suite', 'The following error originated from your test code', @@ -898,83 +729,58 @@ describe('errors ui', { }) }) - describe('custom commands', () => { - const file = 'custom_commands.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:2', - }) + it('custom commands', () => { + const verify = loadSpec({ + fileName: 'custom_commands.cy.js', + onLoadStatsMessage: 'Failed:3', }) - verify.it('assertion failure', { - file, + verify('assertion failure', { column: 23, message: `expected 'actual' to equal 'expected'`, codeFrameText: `add('failAssertion'`, }) - verify.it('exception', { - file, + verify('exception', { column: 8, message: 'bar is not a function', codeFrameText: `add('failException'`, }) - verify.it('command failure', { - file, + verify('command failure', { column: 6, message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', codeFrameText: `add('failCommand'`, }) }) - describe('typescript', () => { - const file = 'typescript.cy.ts' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:3', - }) + it('typescript', () => { + const verify = loadSpec({ + fileName: 'typescript.cy.ts', + onLoadStatsMessage: 'Failed:3', }) - verify.it('assertion failure', { - file, + verify('assertion failure', { column: 25, message: `expected 'actual' to equal 'expected'`, }) - verify.it('exception', { - file, + verify('exception', { column: 10, message: 'bar is not a function', }) - verify.it('command failure', { - file, + verify('command failure', { column: 8, message: 'Timed out retrying after 0ms: Expected to find element: #does-not-exist, but never found it', codeFrameText: `.get('#does-not-exist')`, }) }) - describe('docs url', () => { - const file = 'docs_url.cy.js' - const docsUrl = 'https://on.cypress.io/viewport' - + context('docs url', () => { before(() => { // @ts-ignore window.top.__cySkipValidateConfig = true - - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:2', - }) }) after(() => { @@ -982,60 +788,59 @@ describe('errors ui', { window.top.__cySkipValidateConfig = false }) - verify.it('displays as link in interactive mode', { retries: 1 }, { - file, - verifyFn () { - cy.contains('.runnable-title', 'displays as link in interactive mode') - .closest('.runnable').within(() => { - cy - .get('.runnable-err-message') - .should('not.contain', docsUrl) - .contains('Learn more') - .should('have.attr', 'href', docsUrl) - }) - }, - }) - - verify.it('is text in error message in run mode', { - file, - verifyFn () { - cy.contains('.runnable-title', 'is text in error message in run mode') - .closest('.runnable').within(() => { - cy - .get('.runnable-err-message') - .should('contain', docsUrl) - .contains('Learn more') - .should('not.exist') - }) - }, + it('docs url validation', { retries: 1 }, () => { + const docsUrl = 'https://on.cypress.io/viewport' + + const verify = loadSpec({ + fileName: 'docs_url.cy.js', + onLoadStatsMessage: 'Failed:2', + }) + + verify('displays as link in interactive mode', { + verifyFn () { + cy.contains('.runnable-title', 'displays as link in interactive mode') + .closest('.runnable').within(() => { + cy + .get('.runnable-err-message') + .should('not.contain', docsUrl) + .contains('Learn more') + .should('have.attr', 'href', docsUrl) + }) + }, + }) + + verify('is text in error message in run mode', { + verifyFn () { + cy.contains('.runnable-title', 'is text in error message in run mode') + .closest('.runnable').within(() => { + cy + .get('.runnable-err-message') + .should('contain', docsUrl) + .contains('Learn more') + .should('not.exist') + }) + }, + }) }) }) // cases where there is a bug in Cypress and we should show cypress internals // instead of the invocation stack. we test this by monkey-patching internal // methods to make them throw an error - describe('unexpected errors', () => { - const file = 'unexpected.cy.js' - - before(() => { - setup({ - fileName: file, - mockPreferredEditor: false, - onLoadStatsMessage: 'Failed:1', - }) + it('unexpected errors', () => { + const verify = loadSpec({ + fileName: 'unexpected.cy.js', + onLoadStatsMessage: 'Failed:1', }) // FIXME: the eval doesn't seem to take effect and overwrite the method // so it ends up not failing properly - // @ts-ignore - verify.it.skip('Cypress method error', { - file, - verifyFn: verifyInternalFailure, - method: 'Cypress.LocalStorage._isSpecialKeyword', - }) + // verify('Cypress method error', { + // verifyFn: verifyInternalFailure, + // method: 'Cypress.LocalStorage._isSpecialKeyword', + // }) - verify.it('internal cy error', { - file, + verify('internal cy error', { verifyFn: verifyInternalFailure, method: 'cy.expect', }) diff --git a/packages/app/cypress/e2e/runner/support/verify-failures.ts b/packages/app/cypress/e2e/runner/support/verify-failures.ts index 2e6705c0e2d0..c507cd3e8d39 100644 --- a/packages/app/cypress/e2e/runner/support/verify-failures.ts +++ b/packages/app/cypress/e2e/runner/support/verify-failures.ts @@ -168,28 +168,17 @@ export const verifyFailure = (options) => { } } -const createVerifyTest = (modifier?: string) => { - return (title: string, opts: any, props?: any) => { - if (!props) { - props = opts - opts = null - } - +export const createVerify = ({ fileName, hasPreferredIde }) => { + return (title: string, props?: any) => { props.specTitle ||= title + props.file ||= fileName - const verifyFn = (props.verifyFn || verifyFailure).bind(null, props) + props.hasPreferredIde = hasPreferredIde - return (modifier ? it[modifier] : it)(title, verifyFn) + return (props.verifyFn || verifyFailure).call(null, props) } } -export const verify = { - it: createVerifyTest(), -} - -verify.it['only'] = createVerifyTest('only') -verify.it['skip'] = createVerifyTest('skip') - export const verifyInternalFailure = (props) => { const { specTitle, method, stackMethod } = props From 51f3924faa1068b58fe448bb9493673da3ee8d9f Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Fri, 4 Feb 2022 00:16:20 -0600 Subject: [PATCH 17/64] Couple API tweaks --- .../cypress/e2e/runner/reporter.errors.cy.ts | 47 +++++++++++++------ .../e2e/runner/support/verify-failures.ts | 27 +++++------ 2 files changed, 46 insertions(+), 28 deletions(-) diff --git a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts index 3cf7b33da4ba..e05c4539268d 100644 --- a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts +++ b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts @@ -1,10 +1,28 @@ import { createVerify, verifyInternalFailure } from './support/verify-failures' -const loadSpec = ({ - fileName, - onLoadStatsMessage, - hasPreferredIde = false, -}) => { +type LoadSpecOptions = { + fileName: string + onLoadStatsMessage: string + hasPreferredIde?: boolean +} + +type VerifyFunc = (specTitle: string, verifyOptions: any) => void + +/** + * Navigates to desired spec file within Cypress app and waits for completion. + * Returns scoped verify function to aid inner spec validation. + */ +function loadSpec (options: LoadSpecOptions): VerifyFunc { + const { + fileName, + onLoadStatsMessage, + hasPreferredIde = false, + } = options + + cy.scaffoldProject('runner-e2e-specs') + cy.openProject('runner-e2e-specs') + cy.startAppServer() + cy.withCtx((ctx, options) => { if (options.hasPreferredIde) { // set preferred editor to bypass IDE selection dialog @@ -23,11 +41,11 @@ const loadSpec = ({ ctx.coreData.localSettings.preferences.isSpecsListOpen = false }, { hasPreferredIde }) - cy.startAppServer() - cy.visitApp() - // directly visiting the spec will sometimes hang, going through // specs page first to mitigate + // cy.visitApp(`specs/runner?file=cypress/e2e/errors/${fileName}`) + + cy.visitApp() cy.contains('[data-cy=spec-item]', fileName).click() cy.location().should((location) => { @@ -35,22 +53,20 @@ const loadSpec = ({ }) // Wait for specs to complete - cy.findByLabelText('Stats', { timeout: 30000 }) + cy.findByLabelText('Stats', { timeout: 10000 }) .get('.failed', { timeout: 10000 }).should('have.text', onLoadStatsMessage) + // Return scoped verify function with spec options baked in return createVerify({ fileName, hasPreferredIde }) } describe('errors ui', { viewportHeight: 768, viewportWidth: 1024, + // Limiting tests kept in memory due to large memory cost + // of nested spec snapshots numTestsKeptInMemory: 1, }, () => { - beforeEach(() => { - cy.scaffoldProject('runner-e2e-specs') - cy.openProject('runner-e2e-specs') - }) - it('assertion failures', () => { const verify = loadSpec({ fileName: 'assertions.cy.js', @@ -779,6 +795,9 @@ describe('errors ui', { context('docs url', () => { before(() => { + // docs_url.cy.js manually sets the 'isInteractive' config property in order + // to test both run and open mode behavior. We need to skip the config validation + // here in order for this to be possible. // @ts-ignore window.top.__cySkipValidateConfig = true }) diff --git a/packages/app/cypress/e2e/runner/support/verify-failures.ts b/packages/app/cypress/e2e/runner/support/verify-failures.ts index c507cd3e8d39..ae7817a70058 100644 --- a/packages/app/cypress/e2e/runner/support/verify-failures.ts +++ b/packages/app/cypress/e2e/runner/support/verify-failures.ts @@ -3,14 +3,14 @@ import defaultMessages from '@packages/frontend-shared/src/locales/en-US.json' // Assert that either the the dialog is presented or the mutation is emitted, depending on // whether the test has a preferred IDE defined. -const verifyIdeOpen = ({ file, action, hasPreferredIde }) => { +const verifyIdeOpen = ({ fileName, action, hasPreferredIde }) => { if (hasPreferredIde) { cy.intercept('mutation-OpenFileInIDE', { data: { 'openFileInIDE': true } }).as('OpenIDE') action() cy.wait('@OpenIDE').then(({ request }) => { - expect(request.body.variables.input.absolute).to.include(file) + expect(request.body.variables.input.absolute).to.include(fileName) }) } else { action() @@ -20,7 +20,7 @@ const verifyIdeOpen = ({ file, action, hasPreferredIde }) => { } } -export const verifyFailure = (options) => { +const verifyFailure = (options) => { const { specTitle, hasCodeFrame = true, @@ -32,7 +32,7 @@ export const verifyFailure = (options) => { notInMessage = [], command, stack, - file, + fileName, uncaught = false, uncaughtMessage, } = options @@ -42,7 +42,7 @@ export const verifyFailure = (options) => { codeFrameText = specTitle } - regex = regex || new RegExp(`${file}:${line || '\\d+'}:${column}`) + regex = regex || new RegExp(`${fileName}:${line || '\\d+'}:${column}`) cy.contains('.runnable-title', specTitle).closest('.runnable').as('Root') @@ -113,10 +113,10 @@ export const verifyFailure = (options) => { if (verifyOpenInIde) { verifyIdeOpen({ - file, + fileName, hasPreferredIde, action: () => { - cy.get('@Root').contains('.runnable-err-stack-trace .runnable-err-file-path a', file) + cy.get('@Root').contains('.runnable-err-stack-trace .runnable-err-file-path a', fileName) .click('left') }, }) @@ -158,10 +158,10 @@ export const verifyFailure = (options) => { if (verifyOpenInIde) { verifyIdeOpen({ - file, + fileName, hasPreferredIde, action: () => { - cy.get('@Root').contains('.test-err-code-frame .runnable-err-file-path a', file) + cy.get('@Root').contains('.test-err-code-frame .runnable-err-file-path a', fileName) .click() }, }) @@ -169,13 +169,12 @@ export const verifyFailure = (options) => { } export const createVerify = ({ fileName, hasPreferredIde }) => { - return (title: string, props?: any) => { - props.specTitle ||= title - props.file ||= fileName - + return (specTitle: string, props?: any) => { + props.specTitle ||= specTitle + props.fileName ||= fileName props.hasPreferredIde = hasPreferredIde - return (props.verifyFn || verifyFailure).call(null, props) + ;(props.verifyFn || verifyFailure).call(null, props) } } From fbfa3841ae6f0536db45a4ebb6c8f9fe918322ff Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Sun, 6 Feb 2022 22:28:13 -0600 Subject: [PATCH 18/64] test: Migrating runner hooks specs to app --- .../cypress/e2e/runner/reporter.errors.cy.ts | 149 +++++++----------- .../cypress/e2e/runner/reporter.hooks.cy.ts} | 110 +++++-------- .../cypress/e2e/runner/support/spec-loader.ts | 62 ++++++++ packages/app/src/runner/index.ts | 2 + .../cypress/e2e/hooks/basic.cy.js | 0 .../cypress/e2e/hooks/only.cy.js | 4 + .../cypress/e2e/hooks/rerun.cy.js | 21 +++ .../cypress/e2e/hooks/skip.cy.js | 1 + 8 files changed, 181 insertions(+), 168 deletions(-) rename packages/{runner/cypress/e2e/reporter.hooks.cy.js => app/cypress/e2e/runner/reporter.hooks.cy.ts} (52%) create mode 100644 packages/app/cypress/e2e/runner/support/spec-loader.ts rename packages/runner/cypress/fixtures/hooks/basic_spec.js => system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/basic.cy.js (100%) rename packages/runner/cypress/fixtures/hooks/only_spec.js => system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/only.cy.js (73%) create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/rerun.cy.js rename packages/runner/cypress/fixtures/hooks/skip_spec.js => system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/skip.cy.js (84%) diff --git a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts index e05c4539268d..2379fe8e73ee 100644 --- a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts +++ b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts @@ -1,60 +1,19 @@ +import * as specLoader from './support/spec-loader' import { createVerify, verifyInternalFailure } from './support/verify-failures' -type LoadSpecOptions = { - fileName: string - onLoadStatsMessage: string - hasPreferredIde?: boolean -} - type VerifyFunc = (specTitle: string, verifyOptions: any) => void /** - * Navigates to desired spec file within Cypress app and waits for completion. + * Navigates to desired error spec file within Cypress app and waits for completion. * Returns scoped verify function to aid inner spec validation. */ -function loadSpec (options: LoadSpecOptions): VerifyFunc { +function loadErrorSpec (options: specLoader.LoadSpecOptions): VerifyFunc { const { fileName, - onLoadStatsMessage, hasPreferredIde = false, } = options - cy.scaffoldProject('runner-e2e-specs') - cy.openProject('runner-e2e-specs') - cy.startAppServer() - - cy.withCtx((ctx, options) => { - if (options.hasPreferredIde) { - // set preferred editor to bypass IDE selection dialog - ctx.coreData.localSettings.availableEditors = [ - ...ctx.coreData.localSettings.availableEditors, - { - id: 'test-editor', - binary: '/usr/bin/test-editor', - name: 'Test editor', - }, - ] - - ctx.coreData.localSettings.preferences.preferredEditorBinary = 'test-editor' - } - - ctx.coreData.localSettings.preferences.isSpecsListOpen = false - }, { hasPreferredIde }) - - // directly visiting the spec will sometimes hang, going through - // specs page first to mitigate - // cy.visitApp(`specs/runner?file=cypress/e2e/errors/${fileName}`) - - cy.visitApp() - cy.contains('[data-cy=spec-item]', fileName).click() - - cy.location().should((location) => { - expect(location.hash).to.contain(fileName) - }) - - // Wait for specs to complete - cy.findByLabelText('Stats', { timeout: 10000 }) - .get('.failed', { timeout: 10000 }).should('have.text', onLoadStatsMessage) + specLoader.loadSpec(options) // Return scoped verify function with spec options baked in return createVerify({ fileName, hasPreferredIde }) @@ -68,10 +27,10 @@ describe('errors ui', { numTestsKeptInMemory: 1, }, () => { it('assertion failures', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'assertions.cy.js', hasPreferredIde: true, - onLoadStatsMessage: 'Failed:3', + failCount: 3, }) verify('with expect().', { @@ -94,9 +53,9 @@ describe('errors ui', { }) it('assertion failures - no preferred IDE', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'assertions.cy.js', - onLoadStatsMessage: 'Failed:3', + failCount: 3, }) verify('with expect().', { @@ -110,9 +69,9 @@ describe('errors ui', { }) it('exception failures', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'exceptions.cy.js', - onLoadStatsMessage: 'Failed:2', + failCount: 2, }) verify('in spec file', { @@ -128,9 +87,9 @@ describe('errors ui', { }) it('hooks', { viewportHeight: 900 }, () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'hooks.cy.js', - onLoadStatsMessage: 'Failed:1', + failCount: 1, }) // https://github.com/cypress-io/cypress/issues/8214 @@ -144,9 +103,9 @@ describe('errors ui', { }) it('commands', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'commands.cy.js', - onLoadStatsMessage: 'Failed:2', + failCount: 2, }) verify('failure', { @@ -161,9 +120,9 @@ describe('errors ui', { }) it('cy.then', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'then.cy.js', - onLoadStatsMessage: 'Failed:3', + failCount: 3, }) verify('assertion failure', { @@ -183,9 +142,9 @@ describe('errors ui', { }) it('cy.should', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'should.cy.js', - onLoadStatsMessage: 'Failed:8', + failCount: 8, }) verify('callback assertion failure', { @@ -233,9 +192,9 @@ describe('errors ui', { }) it('cy.each', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'each.cy.js', - onLoadStatsMessage: 'Failed:3', + failCount: 3, }) verify('assertion failure', { @@ -255,9 +214,9 @@ describe('errors ui', { }) it('cy.spread', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'spread.cy.js', - onLoadStatsMessage: 'Failed:3', + failCount: 3, }) verify('assertion failure', { @@ -277,9 +236,9 @@ describe('errors ui', { }) it('cy.within', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'within.cy.js', - onLoadStatsMessage: 'Failed:3', + failCount: 3, }) verify('assertion failure', { @@ -299,9 +258,9 @@ describe('errors ui', { }) it('cy.wrap', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'wrap.cy.js', - onLoadStatsMessage: 'Failed:3', + failCount: 3, }) verify('assertion failure', { @@ -321,9 +280,9 @@ describe('errors ui', { }) it('cy.visit', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'visit.cy.js', - onLoadStatsMessage: 'Failed:3', + failCount: 3, }) verify('onBeforeLoad assertion failure', { @@ -352,9 +311,9 @@ describe('errors ui', { }) it('cy.intercept', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'intercept.cy.ts', - onLoadStatsMessage: 'Failed:3', + failCount: 3, }) verify('assertion failure in request callback', { @@ -395,9 +354,9 @@ describe('errors ui', { }) it('cy.route', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'route.cy.js', - onLoadStatsMessage: 'Failed:9', + failCount: 9, }) verify('callback assertion failure', { @@ -453,9 +412,9 @@ describe('errors ui', { }) it('cy.server', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'server.cy.js', - onLoadStatsMessage: 'Failed:6', + failCount: 6, }) verify('onAbort assertion failure', { @@ -496,9 +455,9 @@ describe('errors ui', { }) it('cy.readFile', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'readfile.cy.js', - onLoadStatsMessage: 'Failed:1', + failCount: 1, }) verify('existence failure', { @@ -508,9 +467,9 @@ describe('errors ui', { }) it('validation errors', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'validation.cy.js', - onLoadStatsMessage: 'Failed:3', + failCount: 3, }) verify('from cypress', { @@ -532,9 +491,9 @@ describe('errors ui', { }) it('event handlers', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'events.cy.js', - onLoadStatsMessage: 'Failed:4', + failCount: 4, }) verify('event assertion failure', { @@ -559,9 +518,9 @@ describe('errors ui', { }) it('uncaught errors', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'uncaught.cy.js', - onLoadStatsMessage: 'Failed:11', + failCount: 11, }) verify('sync app visit exception', { @@ -708,9 +667,9 @@ describe('errors ui', { }) it('uncaught errors: outside test', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'uncaught_outside_test.cy.js', - onLoadStatsMessage: 'Failed:1', + failCount: 1, }) // NOTE: the following 2 test don't have uncaught: true because we don't @@ -729,9 +688,9 @@ describe('errors ui', { }) it('uncaught errors: outside test only suite', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'uncaught_outside_test_only_suite.cy.js', - onLoadStatsMessage: 'Failed:1', + failCount: 1, }) verify('An uncaught error was detected outside of a test', { @@ -746,9 +705,9 @@ describe('errors ui', { }) it('custom commands', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'custom_commands.cy.js', - onLoadStatsMessage: 'Failed:3', + failCount: 3, }) verify('assertion failure', { @@ -771,9 +730,9 @@ describe('errors ui', { }) it('typescript', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'typescript.cy.ts', - onLoadStatsMessage: 'Failed:3', + failCount: 3, }) verify('assertion failure', { @@ -810,9 +769,9 @@ describe('errors ui', { it('docs url validation', { retries: 1 }, () => { const docsUrl = 'https://on.cypress.io/viewport' - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'docs_url.cy.js', - onLoadStatsMessage: 'Failed:2', + failCount: 2, }) verify('displays as link in interactive mode', { @@ -847,9 +806,9 @@ describe('errors ui', { // instead of the invocation stack. we test this by monkey-patching internal // methods to make them throw an error it('unexpected errors', () => { - const verify = loadSpec({ + const verify = loadErrorSpec({ fileName: 'unexpected.cy.js', - onLoadStatsMessage: 'Failed:1', + failCount: 1, }) // FIXME: the eval doesn't seem to take effect and overwrite the method diff --git a/packages/runner/cypress/e2e/reporter.hooks.cy.js b/packages/app/cypress/e2e/runner/reporter.hooks.cy.ts similarity index 52% rename from packages/runner/cypress/e2e/reporter.hooks.cy.js rename to packages/app/cypress/e2e/runner/reporter.hooks.cy.ts index 34638ee1ae42..fc46f9814b25 100644 --- a/packages/runner/cypress/e2e/reporter.hooks.cy.js +++ b/packages/app/cypress/e2e/runner/reporter.hooks.cy.ts @@ -1,12 +1,12 @@ -const helpers = require('../support/helpers') - -const { createCypress } = helpers -const { runIsolatedCypress } = createCypress() +import * as specLoader from './support/spec-loader' describe('hooks', function () { describe('displays hooks', function () { beforeEach(function () { - return runIsolatedCypress(`cypress/fixtures/hooks/basic_spec.js`) + specLoader.loadSpec({ + fileName: 'basic.cy.js', + passCount: 2, + }) }) it('displays commands under correct hook', function () { @@ -16,17 +16,14 @@ describe('hooks', function () { cy.contains('before each').closest('.collapsible').should('contain', 'beforeEachHook 1') cy.contains('test body').closest('.collapsible').should('contain', 'testBody 1') cy.contains('after each').closest('.collapsible').should('contain', 'afterEachHook 1') - }) - - it('displays hooks without number when only one of type', function () { - cy.contains('tests 1').click() + // displays hooks without number when only one of type cy.contains('before all').should('not.contain', '(1)') cy.contains('before each').should('not.contain', '(1)') cy.contains('after each').should('not.contain', '(1)') - }) - it('displays hooks separately with number when more than one of type', function () { + // displays hooks separately with number when more than one of type + cy.contains('tests 1').click() cy.contains('tests 2').click() cy.contains('before all (1)').closest('.collapsible').should('contain', 'beforeHook 2') @@ -43,17 +40,10 @@ describe('hooks', function () { describe('open in IDE', function () { beforeEach(function () { - this.editor = {} - - return runIsolatedCypress(`cypress/fixtures/hooks/basic_spec.js`, { - onBeforeRun: ({ win }) => { - this.win = win - - win.runnerWs.emit.withArgs('get:user:editor') - .yields({ - preferredOpener: this.editor, - }) - }, + specLoader.loadSpec({ + fileName: 'basic.cy.js', + passCount: 2, + hasPreferredIde: true, }) }) @@ -61,46 +51,35 @@ describe('hooks', function () { cy.contains('tests 1').click() cy.get('.hook-open-in-ide').should('have.length', 4) - }) - - it('properly opens file in IDE at hook', function () { - cy.contains('tests 1').click() - cy.contains('Open in IDE').invoke('show').click().then(function () { - expect(this.win.runnerWs.emit.withArgs('open:file').lastCall.args[1].file).to.include('basic_spec.js') - // chrome sets the column to right before "before(" - // while firefox sets it right after "before(" - expect(this.win.runnerWs.emit.withArgs('open:file').lastCall.args[1].column).to.be.eq(Cypress.browser.family === 'firefox' ? 10 : 3) - expect(this.win.runnerWs.emit.withArgs('open:file').lastCall.args[1].line).to.be.eq(2) - }) - }) + cy.intercept('mutation-OpenFileInIDE', { data: { 'openFileInIDE': true } }).as('OpenIDE') - it('properly opens file in IDE at test', function () { - cy.contains('tests 1').click() + cy.contains('Open in IDE').invoke('show').click({ force: true }) - cy.get('a:contains(Open in IDE)').eq(2).invoke('show').click().then(function () { - expect(this.win.runnerWs.emit.withArgs('open:file').lastCall.args[1].file).to.include('basic_spec.js') - // chrome sets the column to right before "it(" - // while firefox sets it right after "it(" - expect(this.win.runnerWs.emit.withArgs('open:file').lastCall.args[1].column).to.be.eq(Cypress.browser.family === 'firefox' ? 6 : 3) - expect(this.win.runnerWs.emit.withArgs('open:file').lastCall.args[1].line).to.be.eq(10) + cy.wait('@OpenIDE').then(({ request }) => { + expect(request.body.variables.input.absolute).to.include('hooks/basic.cy.js') + expect(request.body.variables.input.column).to.eq(Cypress.browser.family === 'firefox' ? 6 : 3) + expect(request.body.variables.input.line).to.eq(2) }) }) }) describe('skipped tests', function () { beforeEach(function () { - return runIsolatedCypress(`cypress/fixtures/hooks/skip_spec.js`) + specLoader.loadSpec({ + fileName: 'skip.cy.js', + passCount: 1, + }) }) it('does not display commands from skipped tests', function () { + // does not display commands from skipped tests cy.contains('test 1').click() - cy.contains('test 1').parents('.collapsible').first().should('not.contain', 'testBody 1') - }) + cy.contains('test 1').click() - // https://github.com/cypress-io/cypress/issues/8086 - it('displays before hook when following it.skip', function () { + // displays before hook when following it.skip + // https://github.com/cypress-io/cypress/issues/8086 cy.contains('test 2').click() cy.contains('test 2').parents('.collapsible').first().should('contain', 'before all') @@ -109,7 +88,10 @@ describe('hooks', function () { describe('only tests', function () { beforeEach(function () { - return runIsolatedCypress(`cypress/fixtures/hooks/only_spec.js`) + specLoader.loadSpec({ + fileName: 'only.cy.js', + passCount: 1, + }) }) it('only displays tests with .only', function () { @@ -133,33 +115,15 @@ describe('hooks', function () { }) // https://github.com/cypress-io/cypress/issues/8189 - it('can rerun without timeout error leaking into next run (due to run restart)', () => { - runIsolatedCypress(() => { - const top = window.parent - - top.count = top.count || 0 - - Cypress.config('defaultCommandTimeout', 50) - afterEach(function () { - assert(true, `run ${top.count}`) + describe('rerun', () => { + it('can rerun without timeout error leaking into next run (due to run restart)', () => { + specLoader.loadSpec({ + fileName: 'rerun.cy.js', + passCount: 1, }) - describe('s1', () => { - it('foo', () => { - cy.once('test:after:run', () => { - if (!top.count) { - requestAnimationFrame(() => { - window.parent.eventManager.reporterBus.emit('runner:restart') - }) - } - - top.count++ - }) - }) - }) + // wait until spec has run twice (due to one reload) + cy.window().its('count').should('eq', 2) }) - - // wait until spec has run twice (due to one reload) - cy.window().its('count').should('eq', 2) }) }) diff --git a/packages/app/cypress/e2e/runner/support/spec-loader.ts b/packages/app/cypress/e2e/runner/support/spec-loader.ts new file mode 100644 index 000000000000..2ae531ff94b8 --- /dev/null +++ b/packages/app/cypress/e2e/runner/support/spec-loader.ts @@ -0,0 +1,62 @@ +export type LoadSpecOptions = { + fileName: string + setup?: () => void + passCount?: number | string + failCount?: number | string + hasPreferredIde?: boolean +} + +export function loadSpec (options: LoadSpecOptions): void { + const { + fileName, + setup, + passCount = '--', + failCount = '--', + hasPreferredIde = false, + } = options + + cy.scaffoldProject('runner-e2e-specs') + cy.openProject('runner-e2e-specs') + cy.startAppServer() + + cy.withCtx((ctx, options) => { + if (options.hasPreferredIde) { + // set preferred editor to bypass IDE selection dialog + ctx.coreData.localSettings.availableEditors = [ + ...ctx.coreData.localSettings.availableEditors, + { + id: 'test-editor', + binary: '/usr/bin/test-editor', + name: 'Test editor', + }, + ] + + ctx.coreData.localSettings.preferences.preferredEditorBinary = 'test-editor' + } + + ctx.coreData.localSettings.preferences.isSpecsListOpen = false + }, { hasPreferredIde }) + + // directly visiting the spec will sometimes hang, going through + // specs page first to mitigate + // cy.visitApp(`specs/runner?file=cypress/e2e/errors/${fileName}`) + + cy.visitApp() + + if (setup) { + setup() + } + + cy.findByLabelText('Search Specs').type(fileName) + cy.contains('[data-cy=spec-item]', fileName).click() + + cy.location().should((location) => { + expect(location.hash).to.contain(fileName) + }) + + // Wait for specs to complete + cy.findByLabelText('Stats', { timeout: 10000 }).within(() => { + cy.get('.passed', { timeout: 10000 }).should('have.text', `Passed:${passCount}`) + cy.get('.failed', { timeout: 10000 }).should('have.text', `Failed:${failCount}`) + }) +} diff --git a/packages/app/src/runner/index.ts b/packages/app/src/runner/index.ts index 35b26688d817..1073c67d0d3c 100644 --- a/packages/app/src/runner/index.ts +++ b/packages/app/src/runner/index.ts @@ -64,6 +64,8 @@ export function getEventManager () { return _eventManager } +window._getEventManager = getEventManager + let _autIframeModel: AutIframe /** diff --git a/packages/runner/cypress/fixtures/hooks/basic_spec.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/basic.cy.js similarity index 100% rename from packages/runner/cypress/fixtures/hooks/basic_spec.js rename to system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/basic.cy.js diff --git a/packages/runner/cypress/fixtures/hooks/only_spec.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/only.cy.js similarity index 73% rename from packages/runner/cypress/fixtures/hooks/only_spec.js rename to system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/only.cy.js index 8bf00531c9c9..b56478e1315e 100644 --- a/packages/runner/cypress/fixtures/hooks/only_spec.js +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/only.cy.js @@ -8,6 +8,8 @@ describe('test wrapper', () => { cy.log('beforeEachHook 1') }) + // Intentional .only for testing + // eslint-disable-next-line mocha/no-exclusive-tests it.only('test 2', () => { cy.log('testBody 2') }) @@ -24,6 +26,8 @@ describe('test wrapper', () => { }) describe('nested suite 3', () => { + // Intentional .only for testing + // eslint-disable-next-line mocha/no-exclusive-tests it.only('test 4', () => { cy.log('testBody 4') }) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/rerun.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/rerun.cy.js new file mode 100644 index 000000000000..028941506068 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/rerun.cy.js @@ -0,0 +1,21 @@ +const top = window.parent + +top.count = top.count || 0 + +describe('s1', { + defaultCommandTimeout: 50, +}, () => { + afterEach(function () { + assert(true, `run ${top.count}`) + }) + + it('foo', () => { + cy.once('test:after:run', () => { + if (!top.count) { + top._getEventManager().reporterBus.emit('runner:restart') + } + + top.count++ + }) + }) +}) diff --git a/packages/runner/cypress/fixtures/hooks/skip_spec.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/skip.cy.js similarity index 84% rename from packages/runner/cypress/fixtures/hooks/skip_spec.js rename to system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/skip.cy.js index 55fac958f4a6..2aa8eecea0cd 100644 --- a/packages/runner/cypress/fixtures/hooks/skip_spec.js +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/skip.cy.js @@ -1,4 +1,5 @@ describe('outer suite', () => { + // NOTE: intentionally skipped for testing it.skip('test 1', () => { cy.log('testBody 1') }) From ea71923c9a7754a160bac1d9e6108bc3bac0fbe8 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Mon, 7 Feb 2022 09:36:12 -0600 Subject: [PATCH 19/64] Removing unused helper from runner package; cleaning up a few things --- .../cypress/e2e/runner/reporter.errors.cy.ts | 2 - .../runner/cypress/support/verify-failures.js | 215 ------------------ .../cypress/e2e/errors/typescript.cy.ts | 4 +- 3 files changed, 1 insertion(+), 220 deletions(-) delete mode 100644 packages/runner/cypress/support/verify-failures.js diff --git a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts index e05c4539268d..26940078aa1f 100644 --- a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts +++ b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts @@ -100,8 +100,6 @@ describe('errors ui', { }) verify('with expect().', { - file: 'assertions.cy.js', - hasPreferredIde: false, column: 25, message: `expected 'actual' to equal 'expected'`, codeFrameText: 'with expect().', diff --git a/packages/runner/cypress/support/verify-failures.js b/packages/runner/cypress/support/verify-failures.js deleted file mode 100644 index 6f0c73c054c1..000000000000 --- a/packages/runner/cypress/support/verify-failures.js +++ /dev/null @@ -1,215 +0,0 @@ -import _ from 'lodash' -import helpers from '../support/helpers' - -const { runIsolatedCypress } = helpers.createCypress({ - config: { isTextTerminal: true, retries: 0 }, - visitUrl: 'http://localhost:3500/fixtures/isolated-runner-inner.html', -}) - -const verifyFailure = (options) => { - const { - hasCodeFrame = true, - verifyOpenInIde = true, - column, - codeFrameText, - originalMessage, - message = [], - notInMessage = [], - command, - stack, - file, - win, - uncaught = false, - uncaughtMessage, - } = options - let { regex, line } = options - - regex = regex || new RegExp(`${file}:${line || '\\d+'}:${column}`) - - const testOpenInIde = () => { - cy.log('open in IDE works').then(() => { - expect(win.runnerWs.emit.withArgs('open:file').lastCall.args[1].file).to.include(file) - }) - } - - win.runnerWs.emit.withArgs('get:user:editor') - .yields({ - preferredOpener: { - id: 'foo-editor', - name: 'Foo', - binary: 'foo-editor', - isOther: false, - }, - }) - - win.runnerWs.emit.withArgs('open:file') - - cy.contains('View stack trace').click() - - const messageLines = [].concat(message) - - if (messageLines.length) { - cy.log('message contains expected lines and stack does not include message') - - _.each(messageLines, (msg) => { - cy.get('.runnable-err-message') - .should('include.text', msg) - - cy.get('.runnable-err-stack-trace') - .should('not.include.text', msg) - }) - } - - if (originalMessage) { - cy.get('.runnable-err-message') - .should('include.text', originalMessage) - } - - const notInMessageLines = [].concat(notInMessage) - - if (notInMessageLines.length) { - cy.log('message does not contain the specified lines') - - _.each(notInMessageLines, (msg) => { - cy.get('.runnable-err-message') - .should('not.include.text', msg) - }) - } - - cy.log('stack trace matches the specified pattern') - cy.get('.runnable-err-stack-trace') - .invoke('text') - .should('match', regex) - - if (stack) { - const stackLines = [].concat(stack) - - if (stackLines.length) { - cy.log('stack contains the expected lines') - } - - _.each(stackLines, (stackLine) => { - cy.get('.runnable-err-stack-trace') - .should('include.text', stackLine) - }) - } - - cy.get('.runnable-err-stack-trace') - .invoke('text') - .should('not.include', '__stackReplacementMarker') - .should((stackTrace) => { - // if this stack trace has the 'From Your Spec Code' addendum, - // it should only appear once - const match = stackTrace.match(/From Your Spec Code/g) - - if (match && match.length) { - expect(match.length, `'From Your Spec Code' should only be in the stack once, but found ${match.length} instances`).to.equal(1) - } - }) - - if (verifyOpenInIde) { - cy.contains('.runnable-err-stack-trace .runnable-err-file-path a', file) - .click('left') - .should(() => { - testOpenInIde() - }) - } - - if (command) { - cy.log('the error is attributed to the correct command') - cy - .get('.command-state-failed') - .first() - .find('.command-method') - .invoke('text') - .should('equal', command) - } - - if (uncaught) { - cy.log('uncaught error has an associated log for the original error') - cy.get('.command-name-uncaught-exception') - .should('have.length', 1) - .should('have.class', 'command-state-failed') - .find('.command-message-text') - .should('include.text', uncaughtMessage || originalMessage) - } else { - cy.log('"caught" error does not have an uncaught error log') - cy.get('.command-name-uncaught-exception').should('not.exist') - } - - if (!hasCodeFrame) return - - cy.log('code frame matches specified pattern') - cy - .get('.test-err-code-frame .runnable-err-file-path') - .invoke('text') - .should('match', regex) - - cy.get('.test-err-code-frame pre span').should('include.text', codeFrameText) - - if (verifyOpenInIde) { - cy.contains('.test-err-code-frame .runnable-err-file-path a', file) - .click() - .should(() => { - expect(win.runnerWs.emit.withArgs('open:file')).to.be.calledTwice - testOpenInIde() - }) - } -} - -const createVerifyTest = (modifier) => { - return (title, opts, props) => { - if (!props) { - props = opts - opts = null - } - - const verifyFn = props.verifyFn || verifyFailure - - const args = _.compact([title, opts, () => { - return runIsolatedCypress(`cypress/fixtures/errors/${props.file}`, { - visitUrl: props.visitUrl, - onBeforeRun ({ specWindow, win, autCypress }) { - specWindow.testToRun = title - specWindow.autWindow = win - specWindow.autCypress = autCypress - - if (props.onBeforeRun) { - props.onBeforeRun({ specWindow, win }) - } - }, - }) - .then(({ win }) => { - props.codeFrameText = props.codeFrameText || title - props.win = win - verifyFn(props, verifyFailure) - }) - }]) - -;(modifier ? it[modifier] : it)(...args) - } -} - -export const verify = { - it: createVerifyTest(), -} - -verify.it['only'] = createVerifyTest('only') -verify.it['skip'] = createVerifyTest('skip') - -export const verifyInternalFailure = (props) => { - const { method, stackMethod } = props - - cy.get('.runnable-err-message') - .should('include.text', `thrown in ${method.replace(/\./g, '-')}`) - - cy.get('.runnable-err-stack-expander > .collapsible-header').click() - - cy.get('.runnable-err-stack-trace') - .should('include.text', stackMethod || method) - - // this is an internal cypress error and we can only show code frames - // from specs, so it should not show the code frame - cy.get('.test-err-code-frame') - .should('not.exist') -} diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/typescript.cy.ts b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/typescript.cy.ts index 4ba57ae4f2bc..96e161ed0a4f 100644 --- a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/typescript.cy.ts +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/typescript.cy.ts @@ -1,4 +1,5 @@ // simple example of typescript types +// eslint-disable-next-line @typescript-eslint/no-unused-vars type Foo = { something: string } @@ -14,9 +15,6 @@ describe('typescript', { defaultCommandTimeout: 0 }, () => { }) it('command failure', () => { - const myVar = { something: 'foo' } as Foo - - cy.log(`Printing TS: ${ myVar}`) cy.get('#does-not-exist') }) }) From 68905974b0cc518e463112154584950faa10969d Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Mon, 7 Feb 2022 10:52:50 -0600 Subject: [PATCH 20/64] Updating yarn.lock --- yarn.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn.lock b/yarn.lock index 78d5c09380da..58defd1c2fdb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17188,9 +17188,9 @@ cyclist@^1.0.1: resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9" integrity sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk= -cypress-example-kitchensink@cypress-io/cypress-example-kitchensink#8389bb6b09c59c2720ceeaadecc8eaa2d4d6551d: +cypress-example-kitchensink@cypress-io/cypress-example-kitchensink#feat/use-supportFiles: version "0.0.0-development" - resolved "https://codeload.github.com/cypress-io/cypress-example-kitchensink/tar.gz/8389bb6b09c59c2720ceeaadecc8eaa2d4d6551d" + resolved "https://codeload.github.com/cypress-io/cypress-example-kitchensink/tar.gz/03d4884a83fe5687e866f747768485ff40e61911" dependencies: npm-run-all "^4.1.2" serve "11.3.0" From f0433c5917eb2ced7c7d437bf5222aac56f05853 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Mon, 7 Feb 2022 13:42:52 -0600 Subject: [PATCH 21/64] fix: Migrating runner sessions ui tests to app --- .../cypress/e2e/runner/reporter.hooks.cy.ts | 30 +++--- .../app/cypress/e2e/runner/sessions.ui.cy.ts | 67 +++++++++++++ .../cypress/e2e/support/e2eSupport.ts | 8 +- packages/runner/cypress/e2e/sessions.ui.cy.js | 96 ------------------- .../cypress/e2e/sessions/blank_session.cy.js | 8 ++ .../e2e/sessions/multiple_sessions.cy.js | 14 +++ .../e2e/sessions/recreated_session.cy.js | 31 ++++++ 7 files changed, 144 insertions(+), 110 deletions(-) create mode 100644 packages/app/cypress/e2e/runner/sessions.ui.cy.ts delete mode 100644 packages/runner/cypress/e2e/sessions.ui.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/sessions/blank_session.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/sessions/multiple_sessions.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/sessions/recreated_session.cy.js diff --git a/packages/app/cypress/e2e/runner/reporter.hooks.cy.ts b/packages/app/cypress/e2e/runner/reporter.hooks.cy.ts index fc46f9814b25..12c82f9f4913 100644 --- a/packages/app/cypress/e2e/runner/reporter.hooks.cy.ts +++ b/packages/app/cypress/e2e/runner/reporter.hooks.cy.ts @@ -1,15 +1,19 @@ import * as specLoader from './support/spec-loader' -describe('hooks', function () { - describe('displays hooks', function () { - beforeEach(function () { +describe('hooks', { + // Limiting tests kept in memory due to large memory cost + // of nested spec snapshots + numTestsKeptInMemory: 1, +}, () => { + describe('displays hooks', () => { + beforeEach(() => { specLoader.loadSpec({ fileName: 'basic.cy.js', passCount: 2, }) }) - it('displays commands under correct hook', function () { + it('displays commands under correct hook', () => { cy.contains('tests 1').click() cy.contains('before all').closest('.collapsible').should('contain', 'beforeHook 1') @@ -38,8 +42,8 @@ describe('hooks', function () { }) }) - describe('open in IDE', function () { - beforeEach(function () { + describe('open in IDE', () => { + beforeEach(() => { specLoader.loadSpec({ fileName: 'basic.cy.js', passCount: 2, @@ -47,7 +51,7 @@ describe('hooks', function () { }) }) - it('creates open in IDE button', function () { + it('creates open in IDE button', () => { cy.contains('tests 1').click() cy.get('.hook-open-in-ide').should('have.length', 4) @@ -64,15 +68,15 @@ describe('hooks', function () { }) }) - describe('skipped tests', function () { - beforeEach(function () { + describe('skipped tests', () => { + beforeEach(() => { specLoader.loadSpec({ fileName: 'skip.cy.js', passCount: 1, }) }) - it('does not display commands from skipped tests', function () { + it('does not display commands from skipped tests', () => { // does not display commands from skipped tests cy.contains('test 1').click() cy.contains('test 1').parents('.collapsible').first().should('not.contain', 'testBody 1') @@ -86,15 +90,15 @@ describe('hooks', function () { }) }) - describe('only tests', function () { - beforeEach(function () { + describe('only tests', () => { + beforeEach(() => { specLoader.loadSpec({ fileName: 'only.cy.js', passCount: 1, }) }) - it('only displays tests with .only', function () { + it('only displays tests with .only', () => { cy.contains('test wrapper').parents('.collapsible').first().should(($suite) => { expect($suite).not.to.contain('test 1') expect($suite).to.contain('nested suite 1') diff --git a/packages/app/cypress/e2e/runner/sessions.ui.cy.ts b/packages/app/cypress/e2e/runner/sessions.ui.cy.ts new file mode 100644 index 000000000000..447edada7d89 --- /dev/null +++ b/packages/app/cypress/e2e/runner/sessions.ui.cy.ts @@ -0,0 +1,67 @@ +import * as specLoader from './support/spec-loader' + +describe('sessions ui', { + viewportWidth: 1000, + viewportHeight: 660, + // Limiting tests kept in memory due to large memory cost + // of nested spec snapshots + numTestsKeptInMemory: 1, +}, () => { + it('empty session with no data', () => { + specLoader.loadSpec({ + fileName: 'blank_session.cy.js', + passCount: 1, + }) + + cy.get('.sessions-container').click() + .should('contain', 'blank_session') + + cy.percySnapshot() + }) + + it('shows message for new, saved, and recreated session', () => { + specLoader.loadSpec({ + fileName: 'recreated_session.cy.js', + passCount: 3, + }) + + cy.get('.test').each(($el) => cy.wrap($el).click()) + + cy.get('.sessions-container').eq(0).click() + .should('contain', '1') + + cy.get('.sessions-container').eq(1).click() + .should('contain', '1') + + cy.get('.test').eq(0) + .should('contain', 'Sessions (1)') + .should('contain', 'user1') + .should('contain', '(new)') + + cy.get('.test').eq(1) + .should('contain', 'Sessions (1)') + .should('contain', 'user1') + .should('contain', '(saved)') + + cy.get('.test').eq(2) + .should('contain', 'Sessions (1)') + .should('contain', 'user1') + .should('contain', '(recreated)') + + cy.percySnapshot() + }) + + it('multiple sessions in a test', () => { + specLoader.loadSpec({ + fileName: 'multiple_sessions.cy.js', + passCount: 1, + }) + + cy.get('.sessions-container').first().click() + .should('contain', 'Sessions (2)') + .should('contain', 'user1') + .should('contain', 'user2') + + cy.percySnapshot() + }) +}) diff --git a/packages/frontend-shared/cypress/e2e/support/e2eSupport.ts b/packages/frontend-shared/cypress/e2e/support/e2eSupport.ts index ed7418da09c6..9b404fc1268d 100644 --- a/packages/frontend-shared/cypress/e2e/support/e2eSupport.ts +++ b/packages/frontend-shared/cypress/e2e/support/e2eSupport.ts @@ -436,5 +436,11 @@ Cypress.Commands.add('remoteGraphQLIntercept', remoteGraphQLIntercept) Cypress.Commands.add('findBrowsers', findBrowsers) Cypress.Commands.add('validateExternalLink', { prevSubject: ['optional', 'element'] }, validateExternalLink) -installCustomPercyCommand() +installCustomPercyCommand({ + elementOverrides: { + '.runnable-header .duration': ($el) => $el.text('XX:XX'), + '.cy-tooltip': true, + }, +}) + addNetworkCommands() diff --git a/packages/runner/cypress/e2e/sessions.ui.cy.js b/packages/runner/cypress/e2e/sessions.ui.cy.js deleted file mode 100644 index 57996a07aefd..000000000000 --- a/packages/runner/cypress/e2e/sessions.ui.cy.js +++ /dev/null @@ -1,96 +0,0 @@ -const helpers = require('../support/helpers') - -const { runIsolatedCypress } = helpers.createCypress({ config: { experimentalSessionSupport: true } }) - -describe('runner/cypress sessions.ui.spec', { viewportWidth: 1000, viewportHeight: 660 }, () => { - it('empty session with no data', () => { - runIsolatedCypress(() => { - it('t1', () => { - cy.session('blank_session', () => {}) - assert(true) - }) - }) - - cy.get('.sessions-container').click() - .should('contain', 'blank_session') - - cy.percySnapshot() - }) - - it('shows message for new, saved, and recreated session', () => { - runIsolatedCypress(() => { - const stub = Cypress.sinon.stub().callsFake(() => { - if (stub.callCount === 3) { - return false - } - }) - - beforeEach(() => { - cy.session('user1', () => { - window.localStorage.foo = 'val' - }, { - validate: stub, - }) - }) - - it('t1', () => { - assert(true) - }) - - it('t2', () => { - assert(true) - }) - - it('t3', () => { - assert(true) - }) - }) - - cy.get('.test').each(($el) => cy.wrap($el).click()) - - cy.get('.sessions-container').eq(0).click() - .should('contain', '1') - - cy.get('.sessions-container').eq(1).click() - .should('contain', '1') - - cy.get('.test').eq(0) - .should('contain', 'Sessions (1)') - .should('contain', 'user1') - .should('contain', '(new)') - - cy.get('.test').eq(1) - .should('contain', 'Sessions (1)') - .should('contain', 'user1') - .should('contain', '(saved)') - - cy.get('.test').eq(2) - .should('contain', 'Sessions (1)') - .should('contain', 'user1') - .should('contain', '(recreated)') - - cy.percySnapshot() - }) - - it('multiple sessions in a test', () => { - runIsolatedCypress(() => { - it('t1', () => { - cy.session('user1', () => { - window.localStorage.foo = 'val' - }) - - cy.session('user2', () => { - window.localStorage.foo = 'val' - window.localStorage.bar = 'val' - }) - }) - }) - - cy.get('.sessions-container').first().click() - .should('contain', 'Sessions (2)') - .should('contain', 'user1') - .should('contain', 'user2') - - cy.percySnapshot() - }) -}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/sessions/blank_session.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/sessions/blank_session.cy.js new file mode 100644 index 000000000000..b12f18ecc383 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/sessions/blank_session.cy.js @@ -0,0 +1,8 @@ +describe('session', { + experimentalSessionSupport: true, +}, () => { + it('creates blank session', () => { + cy.session('blank_session', () => {}) + assert(true) + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/sessions/multiple_sessions.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/sessions/multiple_sessions.cy.js new file mode 100644 index 000000000000..35adcb36ea83 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/sessions/multiple_sessions.cy.js @@ -0,0 +1,14 @@ +describe('recreated session', { + experimentalSessionSupport: true, +}, () => { + it('t1', () => { + cy.session('user1', () => { + window.localStorage.foo = 'val' + }) + + cy.session('user2', () => { + window.localStorage.foo = 'val' + window.localStorage.bar = 'val' + }) + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/sessions/recreated_session.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/sessions/recreated_session.cy.js new file mode 100644 index 000000000000..72259a345e52 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/sessions/recreated_session.cy.js @@ -0,0 +1,31 @@ +describe('recreated session', { + experimentalSessionSupport: true, +}, () => { + const stub = Cypress.sinon.stub().callsFake(() => { + // The validation for t3 will fail, causing the + // session to be recreated (rather than load from saved) + if (stub.callCount === 3) { + return false + } + }) + + beforeEach(() => { + cy.session('user1', () => { + window.localStorage.foo = 'val' + }, { + validate: stub, + }) + }) + + it('t1', () => { + assert(true) + }) + + it('t2', () => { + assert(true) + }) + + it('t3', () => { + assert(true) + }) +}) From 340a82936e91b75e25235810ae552de71c0d1368 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Mon, 7 Feb 2022 14:47:48 -0600 Subject: [PATCH 22/64] test: Migrating runner's runner.ui tests to app --- .../app/cypress/e2e/runner/runner.ui.cy.ts | 245 ++++++++++++++++++ .../cypress/e2e/runner/support/spec-loader.ts | 21 +- .../e2e/runner/async-timeout.runner.cy.js | 20 ++ .../e2e/runner/catch-fail.runner.cy.js | 22 ++ .../runner/disabled-command-log.runner.cy.js | 5 + .../e2e/runner/empty-suites.runner.cy.js | 11 + .../cypress/e2e/runner/fail-pass.runner.cy.js | 15 ++ .../e2e/runner/fail-with-after.runner.cy.js | 10 + .../runner/fail-with-all-hooks.runner.cy.js | 10 + .../e2e/runner/fail-with-before.runner.cy.js | 16 ++ .../e2e/runner/failing-hooks.runner.cy.js | 25 ++ .../e2e/runner/nested-suite.runner.cy.js | 12 + .../cypress/e2e/runner/no-tests.runner.cy.js | 1 + .../runner/pass-fail-pass-fail.runner.cy.js | 24 ++ .../runner/scrolls-command-log.runner.cy.js | 8 + .../e2e/runner/simple-fail.runner.cy.js | 14 + .../simple-single-global-test.runner.cy.js | 3 + .../runner/simple-single-test.runner.cy.js | 5 + .../e2e/runner/stop-execution.runner.cy.js | 13 + .../cypress/e2e/runner/test-only.runner.cy.js | 14 + .../e2e/runner/test-pending.runner.cy.js | 5 + .../runner/three-simple-tests.runner.cy.js | 5 + .../e2e/support/generate-mocha-tests.js | 169 ++++++++++++ 23 files changed, 669 insertions(+), 4 deletions(-) create mode 100644 packages/app/cypress/e2e/runner/runner.ui.cy.ts create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/async-timeout.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/catch-fail.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/disabled-command-log.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/empty-suites.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/fail-pass.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/fail-with-after.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/fail-with-all-hooks.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/fail-with-before.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/failing-hooks.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/nested-suite.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/no-tests.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/pass-fail-pass-fail.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/scrolls-command-log.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/simple-fail.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/simple-single-global-test.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/simple-single-test.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/stop-execution.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/test-only.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/test-pending.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/three-simple-tests.runner.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/support/generate-mocha-tests.js diff --git a/packages/app/cypress/e2e/runner/runner.ui.cy.ts b/packages/app/cypress/e2e/runner/runner.ui.cy.ts new file mode 100644 index 000000000000..aa33f5003e17 --- /dev/null +++ b/packages/app/cypress/e2e/runner/runner.ui.cy.ts @@ -0,0 +1,245 @@ +import * as specLoader from './support/spec-loader' + +describe('src/cypress/runner', () => { + describe('tests finish with correct state', () => { + it('simple 1 test', () => { + specLoader.loadSpec({ + fileName: 'simple-single-test.runner.cy.js', + passCount: 1, + failCount: 0, + }) + }) + + it('simple 1 global test', () => { + specLoader.loadSpec({ + fileName: 'simple-single-global-test.runner.cy.js', + passCount: 1, + failCount: 0, + }) + }) + + it('simple 3 tests', () => { + specLoader.loadSpec({ + fileName: 'three-simple-tests.runner.cy.js', + passCount: 3, + failCount: 0, + }) + }) + + it('simple fail', () => { + specLoader.loadSpec({ + fileName: 'simple-fail.runner.cy.js', + passCount: 0, + failCount: 1, + }) + + // render exactly one error + cy.get('.runnable-err:contains(AssertionError)').should('have.length', 1) + }) + + it('pass fail pass fail', () => { + specLoader.loadSpec({ + fileName: 'pass-fail-pass-fail.runner.cy.js', + passCount: 2, + failCount: 2, + }) + }) + + it('fail pass', () => { + specLoader.loadSpec({ + fileName: 'fail-pass.runner.cy.js', + passCount: 1, + failCount: 1, + }) + }) + + it('no tests', () => { + specLoader.loadSpec({ + fileName: 'no-tests.runner.cy.js', + passCount: 0, + failCount: 0, + }) + + cy.contains('No tests found.').should('be.visible') + cy.contains('p', 'Cypress could not detect tests in this file.').should('be.visible') + }) + + it('executes nested suite', () => { + specLoader.loadSpec({ + fileName: 'nested-suite.runner.cy.js', + passCount: 3, + failCount: 0, + }) + }) + + it('simple fail, catch cy.on(fail)', () => { + specLoader.loadSpec({ + fileName: 'catch-fail.runner.cy.js', + passCount: 1, + failCount: 0, + }) + }) + + describe('hook failures', () => { + describe('test failures w/ hooks', () => { + it('test [only]', () => { + specLoader.loadSpec({ + fileName: 'test-only.runner.cy.js', + passCount: 1, + failCount: 0, + }) + }) + + it('test [pending]', () => { + specLoader.loadSpec({ + fileName: 'test-pending.runner.cy.js', + passCount: 0, + failCount: 0, + pendingCount: 3, + }) + }) + + it('fail with [before]', () => { + specLoader.loadSpec({ + fileName: 'fail-with-before.runner.cy.js', + passCount: 1, + failCount: 1, + }) + }) + + it('fail with [after]', () => { + specLoader.loadSpec({ + fileName: 'fail-with-after.runner.cy.js', + passCount: 1, + failCount: 1, + }) + }) + + it('fail with all hooks', () => { + specLoader.loadSpec({ + fileName: 'fail-with-all-hooks.runner.cy.js', + passCount: 0, + failCount: 1, + }) + }) + }) + }) + }) + + describe('other specs', () => { + it('simple failing hook spec', () => { + specLoader.loadSpec({ + fileName: 'failing-hooks.runner.cy.js', + passCount: 1, + failCount: 3, + pendingCount: 1, + }) + + cy.contains('.test', 'never gets here').should('have.class', 'runnable-failed') + cy.contains('.command', 'beforeEach').should('have.class', 'command-state-failed') + cy.contains('.runnable-err', 'beforeEach').scrollIntoView().should('be.visible') + + cy.contains('.test', 'is pending').should('have.class', 'runnable-pending') + + cy.contains('.test', 'fails this').should('have.class', 'runnable-failed') + cy.contains('.command', 'afterEach').should('have.class', 'command-state-failed') + cy.contains('.runnable-err', 'afterEach').scrollIntoView().should('be.visible') + + cy.contains('.test', 'does not run this').should('have.class', 'runnable-processing') + + cy.contains('.test', 'runs this').should('have.class', 'runnable-passed') + + cy.contains('.test', 'fails on this').should('have.class', 'runnable-failed') + cy.contains('.command', 'after').should('have.class', 'command-state-failed') + cy.contains('.runnable-err', 'after').scrollIntoView().should('be.visible') + }) + + it('async timeout spec', () => { + specLoader.loadSpec({ + fileName: 'async-timeout.runner.cy.js', + passCount: 0, + failCount: 1, + }) + }) + + it('scrolls each command into view', () => { + specLoader.loadSpec({ + fileName: 'scrolls-command-log.runner.cy.js', + passCount: 0, + failCount: 1, + }) + + cy.get('.command-number:contains(25)').should('be.visible') + }) + + it('file with empty suites only displays no tests found', () => { + specLoader.loadSpec({ + fileName: 'empty-suites.runner.cy.js', + passCount: 0, + failCount: 0, + }) + + cy.get('.reporter').contains('No tests found') + }) + }) + + // TODO: determine coverage provided by inflight changes + // describe('runner header', () => { + // context('viewport dropdown', () => { + // it('shows on click', () => { + // runIsolatedCypress({}) + // cy.get('.viewport-menu').should('not.be.visible') + // cy.get('.viewport-info button').click() + // cy.get('.viewport-menu').should('be.visible') + + // cy.percySnapshot() + // }) + // }) + + // context('selector playground', () => { + // it('shows on click', () => { + // runIsolatedCypress({}) + + // cy.get('.selector-playground').should('not.be.visible') + // cy.get('.selector-playground-toggle').click() + // cy.get('.selector-playground').should('be.visible') + + // cy.percySnapshot() + // }) + + // it('closes on restart', () => { + // runIsolatedCypress({}) + + // cy.get('.selector-playground-toggle').click() + // cy.get('.selector-playground').should('be.visible') + + // cy.get('.restart').click() + // cy.get('.selector-playground').should('not.be.visible') + // }) + // }) + // }) + + describe('reporter interaction', () => { + // https://github.com/cypress-io/cypress/issues/8621 + it('user can stop test execution', () => { + specLoader.loadSpec({ + fileName: 'stop-execution.runner.cy.js', + passCount: 0, + failCount: 1, + }) + + cy.get('.runnable-err-message').should('not.contain', 'ran afterEach even though specs were stopped') + }) + + // TODO: determine intended function in new runner + // it('supports disabling command log reporter with env var NO_COMMAND_LOG', () => { + // specLoader.loadSpec({ + // fileName: 'disabled-command-log.runner.cy.js', + // passCount: 0, + // failCount: 0, + // }) + + // cy.get('.reporter').should('not.exist') + // }) + }) +}) diff --git a/packages/app/cypress/e2e/runner/support/spec-loader.ts b/packages/app/cypress/e2e/runner/support/spec-loader.ts index 2ae531ff94b8..ce53827f88d8 100644 --- a/packages/app/cypress/e2e/runner/support/spec-loader.ts +++ b/packages/app/cypress/e2e/runner/support/spec-loader.ts @@ -1,8 +1,23 @@ +export const shouldHaveTestResults = ({ passCount, failCount, pendingCount }) => { + passCount = passCount || '--' + failCount = failCount || '--' + + cy.findByLabelText('Stats', { timeout: 10000 }).within(() => { + cy.get('.passed .num', { timeout: 10000 }).should('have.text', `${passCount}`) + cy.get('.failed .num', { timeout: 10000 }).should('have.text', `${failCount}`) + + if (pendingCount) { + cy.get('.pending .num', { timeout: 10000 }).should('have.text', `${pendingCount}`) + } + }) +} + export type LoadSpecOptions = { fileName: string setup?: () => void passCount?: number | string failCount?: number | string + pendingCount?: number | string hasPreferredIde?: boolean } @@ -13,6 +28,7 @@ export function loadSpec (options: LoadSpecOptions): void { passCount = '--', failCount = '--', hasPreferredIde = false, + pendingCount, } = options cy.scaffoldProject('runner-e2e-specs') @@ -55,8 +71,5 @@ export function loadSpec (options: LoadSpecOptions): void { }) // Wait for specs to complete - cy.findByLabelText('Stats', { timeout: 10000 }).within(() => { - cy.get('.passed', { timeout: 10000 }).should('have.text', `Passed:${passCount}`) - cy.get('.failed', { timeout: 10000 }).should('have.text', `Failed:${failCount}`) - }) + shouldHaveTestResults({ passCount, failCount, pendingCount }) } diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/async-timeout.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/async-timeout.runner.cy.js new file mode 100644 index 000000000000..ecc5ffc1a528 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/async-timeout.runner.cy.js @@ -0,0 +1,20 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'async': { + tests: [{ + name: 'bar fails', + // eslint-disable-next-line + fn (done) { + this.timeout(100) + cy.on('fail', () => {}) + // foo is intentionally undefined + // eslint-disable-next-line + foo.bar() // kaboom + }, + eval: true, + }], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/catch-fail.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/catch-fail.runner.cy.js new file mode 100644 index 000000000000..46d05336f951 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/catch-fail.runner.cy.js @@ -0,0 +1,22 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + tests: [ + { + name: 'test 1', + fn: () => { + cy.on('fail', () => { + return false + }) + + expect(false).ok + throw new Error('error in test') + }, + eval: true, + }, + ], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/disabled-command-log.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/disabled-command-log.runner.cy.js new file mode 100644 index 000000000000..e588839cf601 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/disabled-command-log.runner.cy.js @@ -0,0 +1,5 @@ +it('disabled command log with NO_COMMAND_LOG', { + env: { NO_COMMAND_LOG: '1' }, +}, () => { + assert(true) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/empty-suites.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/empty-suites.runner.cy.js new file mode 100644 index 000000000000..16b6ff2fe075 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/empty-suites.runner.cy.js @@ -0,0 +1,11 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + suites: { + 'suite 2': {}, + }, + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/fail-pass.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/fail-pass.runner.cy.js new file mode 100644 index 000000000000..2a35f8d0d276 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/fail-pass.runner.cy.js @@ -0,0 +1,15 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + tests: [ + { + name: 'test 1', + fail: true, + }, + { name: 'test 2' }, + ], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/fail-with-after.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/fail-with-after.runner.cy.js new file mode 100644 index 000000000000..0c60516dd9b7 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/fail-with-after.runner.cy.js @@ -0,0 +1,10 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + hooks: [{ type: 'after' }], + tests: [{ name: 'test 1', fail: true }, 'test 2'], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/fail-with-all-hooks.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/fail-with-all-hooks.runner.cy.js new file mode 100644 index 000000000000..bce95e47e65c --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/fail-with-all-hooks.runner.cy.js @@ -0,0 +1,10 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + hooks: ['before', 'beforeEach', 'afterEach', 'after'], + tests: [{ name: 'test 1', fail: true }], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/fail-with-before.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/fail-with-before.runner.cy.js new file mode 100644 index 000000000000..34227edde9d2 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/fail-with-before.runner.cy.js @@ -0,0 +1,16 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + hooks: ['before'], + tests: [ + { + name: 'test 1', + fail: true, + }, + { name: 'test 2' }, + ], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/failing-hooks.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/failing-hooks.runner.cy.js new file mode 100644 index 000000000000..8035a96dc8ba --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/failing-hooks.runner.cy.js @@ -0,0 +1,25 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'simple failing hook spec': { + suites: { + 'beforeEach hooks': { + hooks: [{ type: 'beforeEach', fail: true }], + tests: ['never gets here'], + }, + 'pending': { + tests: [{ name: 'is pending', pending: true }], + }, + 'afterEach hooks': { + hooks: [{ type: 'afterEach', fail: true }], + tests: ['fails this', 'does not run this'], + }, + 'after hooks': { + hooks: [{ type: 'after', fail: true }] + , tests: ['runs this', 'fails on this'], + }, + }, + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/nested-suite.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/nested-suite.runner.cy.js new file mode 100644 index 000000000000..f3bbb19a44ad --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/nested-suite.runner.cy.js @@ -0,0 +1,12 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { tests: ['test 1', 'test 2'], + suites: { + 'suite 1-1': { + tests: ['test 1'], + }, + } }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/no-tests.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/no-tests.runner.cy.js new file mode 100644 index 000000000000..343ec7bbaa7d --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/no-tests.runner.cy.js @@ -0,0 +1 @@ +// no tests here diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/pass-fail-pass-fail.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/pass-fail-pass-fail.runner.cy.js new file mode 100644 index 000000000000..edaebb32992b --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/pass-fail-pass-fail.runner.cy.js @@ -0,0 +1,24 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + tests: [ + 'test 1', + { + name: 'test 2', + fail: true, + }, + ], + }, + 'suite 2': { + tests: [ + 'test 1', + { + name: 'test 2', + fail: true, + }, + ], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/scrolls-command-log.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/scrolls-command-log.runner.cy.js new file mode 100644 index 000000000000..757e81270c19 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/scrolls-command-log.runner.cy.js @@ -0,0 +1,8 @@ +describe('s1', () => { + // Passing in done forces the spec to timeout + // eslint-disable-next-line + it('t1', (done) => { + cy.timeout(10) + Cypress._.times(25, () => expect(true).ok) + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/simple-fail.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/simple-fail.runner.cy.js new file mode 100644 index 000000000000..24e03509f46b --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/simple-fail.runner.cy.js @@ -0,0 +1,14 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + tests: [ + { + name: 'test 1', + fail: true, + }, + ], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/simple-single-global-test.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/simple-single-global-test.runner.cy.js new file mode 100644 index 000000000000..5aa93169693f --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/simple-single-global-test.runner.cy.js @@ -0,0 +1,3 @@ +it('foo', () => { + expect(true).is.true +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/simple-single-test.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/simple-single-test.runner.cy.js new file mode 100644 index 000000000000..cf883d41c616 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/simple-single-test.runner.cy.js @@ -0,0 +1,5 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { 'suite 1': { tests: [{ name: 'test 1' }] } }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/stop-execution.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/stop-execution.runner.cy.js new file mode 100644 index 000000000000..26109c8363f9 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/stop-execution.runner.cy.js @@ -0,0 +1,13 @@ +describe('stop execution', () => { + it('test stops while running', () => { + cy.timeout(200) + cy.get('.not-exist') + setTimeout(() => { + cy.$$('button.stop', parent.document).click() + }, 100) + }) + + afterEach(function () { + this.currentTest.err = new Error('ran afterEach even though specs were stopped') + }) +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/test-only.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/test-only.runner.cy.js new file mode 100644 index 000000000000..f1c55aab6740 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/test-only.runner.cy.js @@ -0,0 +1,14 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + hooks: ['before', 'beforeEach', 'afterEach', 'after'], + tests: [ + { name: 'test 1' }, + { name: 'test 2', only: true }, + { name: 'test 3' }, + ], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/test-pending.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/test-pending.runner.cy.js new file mode 100644 index 000000000000..bc889d789e67 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/test-pending.runner.cy.js @@ -0,0 +1,5 @@ +before(() => {}) +it('t1') +it('t2') +it('t3') +after(() => {}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/three-simple-tests.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/three-simple-tests.runner.cy.js new file mode 100644 index 000000000000..34d10677244b --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/three-simple-tests.runner.cy.js @@ -0,0 +1,5 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { 'suite 1': { tests: ['test 1', 'test 2', 'test 3'] } }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/support/generate-mocha-tests.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/support/generate-mocha-tests.js new file mode 100644 index 000000000000..216196b85b98 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/support/generate-mocha-tests.js @@ -0,0 +1,169 @@ +const debug = require('debug')('spec') +const _ = Cypress._ + +const evalFn = (win, fn) => { + return function () { + return win.eval(`(${fn.toString()})`).call(this) + } +} + +const createHooks = (win, hooks = []) => { + _.each(hooks, (hook) => { + if (_.isString(hook)) { + hook = { type: hook } + } + + let { type, fail, fn, agents } = hook + + if (fn) { + if (hook.eval) { + const fnStr = fn.toString() + + const newFn = function () { + return win.eval(`(${fnStr})`).call(this) + } + + Object.defineProperty(newFn, 'length', { value: fn.length }) + fn = newFn + } + + return win[type](fn) + } + + if (fail) { + const numFailures = fail + + return win[type](function () { + const message = `${type} - ${this._runnable.parent.title || 'root'}` + + if (agents) { + registerAgents(win) + } + + if (_.isNumber(fail) && fail-- <= 0) { + debug(`hook pass after (${numFailures}) failures: ${type}`) + win.assert(true, message) + + return + } + + if (agents) { + failCypressCommand(win, type) + } else { + debug(`hook fail: ${type}`) + + win.assert(false, message) + + throw new Error(`hook failed: ${type}`) + } + }) + } + + return win[type](function () { + win.assert(true, `${type} - ${this._runnable.parent.title || 'root'}`) + debug(`hook pass: ${type}`) + }) + }) +} + +const createTests = (win, tests = []) => { + _.each(tests, (test) => { + if (_.isString(test)) { + test = { name: test } + } + + let { name, pending, fail, fn, only, agents } = test + + let it = win.it + + if (only) { + it = it['only'] + } + + if (fn) { + if (test.eval) { + const fnStr = fn.toString() + + const newFn = function () { + return win.eval(`(${fnStr})`).call(this) + } + + Object.defineProperty(newFn, 'length', { value: fn.length }) + fn = newFn + } + + return it(name, fn) + } + + if (pending) { + return it(name) + } + + if (fail) { + return it(name, () => { + if (agents) { + registerAgents(win) + } + + if (_.isNumber(fail) && fail-- === 0) { + debug(`test pass after retry: ${name}`) + win.assert(true, name) + + return + } + + if (agents) { + failCypressCommand(win, name) + } else { + debug(`test fail: ${name}`) + win.assert(false, name) + + throw new Error(`test fail: ${name}`) + } + }) + } + + return it(name, () => { + debug(`test pass: ${name}`) + win.assert(true, name) + }) + }) +} + +const failCypressCommand = (win, name) => win.cy.wrap(name).then(() => win.assert(false, name)) +const registerAgents = (win) => { + const obj = { foo: 'bar' } + + win.cy.stub(obj, 'foo') + win.cy.wrap(obj).should('exist') + win.cy.server() + win.cy.route('https://example.com') +} + +export const createSuites = (win, suites = {}) => { + _.each(suites, (obj, suiteName) => { + let fn = () => { + createHooks(win, obj.hooks) + createTests(win, obj.tests) + createSuites(win, obj.suites) + } + + if (_.isFunction(obj)) { + fn = evalFn(win, obj) + } + + win.describe(suiteName, fn) + }) +} + +export const generateMochaTestsForWin = (win, obj) => { + if (typeof obj === 'function') { + win.eval(`( ${obj.toString()})()`) + + return + } + + createHooks(win, obj.hooks) + createTests(win, obj.tests) + createSuites(win, obj.suites) +} From 7d1e7383fb10defe550c6d49d8a5c7fc5f947021 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Wed, 9 Feb 2022 01:56:36 -0600 Subject: [PATCH 23/64] test: Migrating runner retries ui tests to app --- .../cypress/e2e/runner/reporter.hooks.cy.ts | 12 +- .../app/cypress/e2e/runner/retries.ui.cy.ts | 370 +++++++++++++ .../app/cypress/e2e/runner/runner.ui.cy.ts | 44 +- .../app/cypress/e2e/runner/sessions.ui.cy.ts | 8 +- packages/runner/cypress/e2e/retries.ui.cy.js | 497 ------------------ packages/runner/cypress/e2e/runner.ui.cy.js | 408 -------------- .../retries/after-all-failure.retries.cy.js | 18 + .../e2e/retries/after-all-once.retries.cy.js | 20 + .../retries/after-each-failure.retries.cy.js | 14 + .../e2e/retries/after-each-skip.retries.cy.js | 13 + .../all-retry-one-failure.retries.cy.js | 16 + .../before-all-called-once.retries.cy.js | 13 + .../retries/before-all-failure.retries.cy.js | 20 + .../retries/before-each-failure.retries.cy.js | 21 + .../retries/before-each-skip.retries.cy.js | 13 + .../retries/collapse-after-pass.retries.cy.js | 15 + .../collapse-prev-attempts.retries.cy.js | 20 + .../retries/configure-in-hook.retries.cy.js | 14 + .../retries/configure-in-suite.retries.cy.js | 12 + .../retries/configure-in-test.retries.cy.js | 11 + .../retries/configure-retries.retries.cy.js | 17 + .../includes-all-in-attempt.retries.cy.js | 13 + .../cypress/e2e/retries/only.retries.cy.js | 17 + ...opens-attempt-for-screenshot.retries.cy.js | 25 + .../e2e/support/generate-mocha-tests.js | 4 +- 25 files changed, 696 insertions(+), 939 deletions(-) create mode 100644 packages/app/cypress/e2e/runner/retries.ui.cy.ts delete mode 100644 packages/runner/cypress/e2e/retries.ui.cy.js delete mode 100644 packages/runner/cypress/e2e/runner.ui.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/after-all-failure.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/after-all-once.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/after-each-failure.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/after-each-skip.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/all-retry-one-failure.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/before-all-called-once.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/before-all-failure.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/before-each-failure.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/before-each-skip.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/collapse-after-pass.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/collapse-prev-attempts.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/configure-in-hook.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/configure-in-suite.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/configure-in-test.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/configure-retries.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/includes-all-in-attempt.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/only.retries.cy.js create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/retries/opens-attempt-for-screenshot.retries.cy.js diff --git a/packages/app/cypress/e2e/runner/reporter.hooks.cy.ts b/packages/app/cypress/e2e/runner/reporter.hooks.cy.ts index 12c82f9f4913..198fc4885b7b 100644 --- a/packages/app/cypress/e2e/runner/reporter.hooks.cy.ts +++ b/packages/app/cypress/e2e/runner/reporter.hooks.cy.ts @@ -1,4 +1,4 @@ -import * as specLoader from './support/spec-loader' +import { loadSpec } from './support/spec-loader' describe('hooks', { // Limiting tests kept in memory due to large memory cost @@ -7,7 +7,7 @@ describe('hooks', { }, () => { describe('displays hooks', () => { beforeEach(() => { - specLoader.loadSpec({ + loadSpec({ fileName: 'basic.cy.js', passCount: 2, }) @@ -44,7 +44,7 @@ describe('hooks', { describe('open in IDE', () => { beforeEach(() => { - specLoader.loadSpec({ + loadSpec({ fileName: 'basic.cy.js', passCount: 2, hasPreferredIde: true, @@ -70,7 +70,7 @@ describe('hooks', { describe('skipped tests', () => { beforeEach(() => { - specLoader.loadSpec({ + loadSpec({ fileName: 'skip.cy.js', passCount: 1, }) @@ -92,7 +92,7 @@ describe('hooks', { describe('only tests', () => { beforeEach(() => { - specLoader.loadSpec({ + loadSpec({ fileName: 'only.cy.js', passCount: 1, }) @@ -121,7 +121,7 @@ describe('hooks', { // https://github.com/cypress-io/cypress/issues/8189 describe('rerun', () => { it('can rerun without timeout error leaking into next run (due to run restart)', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'rerun.cy.js', passCount: 1, }) diff --git a/packages/app/cypress/e2e/runner/retries.ui.cy.ts b/packages/app/cypress/e2e/runner/retries.ui.cy.ts new file mode 100644 index 000000000000..54484a3b02c7 --- /dev/null +++ b/packages/app/cypress/e2e/runner/retries.ui.cy.ts @@ -0,0 +1,370 @@ +import { loadSpec } from './support/spec-loader' + +const getAttemptTag = (sel) => { + return cy.get(`.test.runnable:contains(${sel}) .attempt-tag`) +} + +const shouldBeOpen = ($el) => cy.wrap($el).parentsUntil('.collapsible').last().parent().should('have.class', 'is-open') + +const attemptTag = (sel) => `.attempt-tag:contains(Attempt ${sel})` + +const containText = (text) => { + return (($el) => { + expect($el[0]).property('innerText').contain(text) + }) +} + +// const cyReject = (fn) => { +// return () => { +// try { +// fn() +// } catch (e) { +// cy.state('reject')(e) +// } +// } +// } + +describe('retries ui', { + viewportWidth: 1024, + viewportHeight: 900, + numTestsKeptInMemory: 1, +}, () => { + it('collapses tests that retry and pass', () => { + loadSpec({ + fileName: 'collapse-after-pass.retries.cy.js', + passCount: 2, + failCount: 0, + }) + + cy.get('.test').should('have.length', 2) + cy.percySnapshot() + }) + + it('collapses prev attempts and keeps final one open on failure', () => { + loadSpec({ + fileName: 'collapse-prev-attempts.retries.cy.js', + passCount: 1, + failCount: 1, + }) + + cy.get('.runnable-err-print').should('be.visible') + cy.percySnapshot() + }) + + it('can toggle failed prev attempt open and log its error', () => { + loadSpec({ + fileName: 'all-retry-one-failure.retries.cy.js', + passCount: 2, + failCount: 1, + }) + + cy.window().then((win) => { + cy.spy(win.console, 'error').as('console_error') + }) + + cy.contains('Attempt 1') + .click() + .closest('.attempt-item') + .find('.runnable-err-print') + .click() + + cy.get('@console_error').should('be.calledWithMatch', 'AssertionError: test 2') + + cy.percySnapshot() + }) + + // TODO: fix this one + // it.only('opens attempt on each attempt failure for the screenshot, and closes after test passes', () => { + // let stub + + // loadSpec({ + // fileName: 'opens-attempt-for-screenshot.retries.cy.js', + // passCount: 3, + // failCount: 0, + // setup: () => { + // let attempt = 0 + + // stub = cy.stub().callsFake(cyReject(() => { + // attempt++ + // expect(cy.$$('.attempt-item > .is-open').length).to.equal(attempt) + // })) + + // debugger + // cy.window().then((win) => { + // debugger + // win.Cypress.Screenshot.onAfterScreenshot = stub + // }) + // }, + // }) + + // cy.get('.test.runnable:contains(t2)').then(($el) => { + // expect($el).not.class('is-open') + // expect(stub).callCount(3) + // }) + // }) + + it('includes routes, spies, hooks, and commands in attempt', () => { + loadSpec({ + fileName: 'includes-all-in-attempt.retries.cy.js', + passCount: 1, + failCount: 0, + }) + + cy.get(attemptTag(1)).click().parentsUntil('.collapsible').last().parent().within(() => { + cy.get('.instruments-container').should('contain', 'Spies / Stubs (1)') + cy.get('.instruments-container').should('contain', 'Routes (1)') + cy.get('.runnable-err').should('contain', 'AssertionError') + }) + + cy.get(attemptTag(2)).click().parentsUntil('.collapsible').last().parent().within(() => { + cy.get('.instruments-container').should('contain', 'Spies / Stubs (2)') + cy.get('.instruments-container').should('contain', 'Routes (2)') + cy.get('.runnable-err').should('contain', 'AssertionError') + }) + + cy.get(attemptTag(3)).parentsUntil('.collapsible').last().parent().within(() => { + cy.get('.instruments-container').should('contain', 'Spies / Stubs (2)') + cy.get('.instruments-container').should('contain', 'Routes (2)') + cy.get('.runnable-err').should('not.be.visible') + }) + + cy.percySnapshot() + }) + + describe('only', () => { + it('test retry with [only]', () => { + loadSpec({ + fileName: 'only.retries.cy.js', + passCount: 1, + failCount: 0, + }) + + cy.contains('test 2') + cy.contains('test 1').should('not.exist') + cy.contains('test 3').should('not.exist') + + cy.percySnapshot() + }) + }) + + describe('beforeAll', () => { + // TODO: make beforeAll hooks retry + it('tests do not retry when beforeAll fails', () => { + loadSpec({ + fileName: 'before-all-failure.retries.cy.js', + passCount: 0, + failCount: 1, + }) + + cy.contains('Although you have test retries') + + cy.percySnapshot() + }) + + // TODO: future versions should run all hooks associated with test on retry + it('before all hooks are not run on the second attempt when fails outside of beforeAll', () => { + loadSpec({ + fileName: 'before-all-called-once.retries.cy.js', + passCount: 1, + failCount: 0, + }) + + cy.contains('test') + cy.contains('after all') + cy.contains('before all').should('not.exist') + + cy.percySnapshot() + }) + }) + + describe('beforeEach', () => { + it('beforeEach hooks retry on failure, but only run same-level afterEach hooks', () => { + loadSpec({ + fileName: 'before-each-failure.retries.cy.js', + passCount: 1, + failCount: 0, + }) + + cy.contains('Attempt 1').click() + cy.get('.attempt-1 .hook-item .collapsible:contains(before each)').find('.command-state-failed') + cy.get('.attempt-1 .hook-item .collapsible:contains(before each (2))').should('not.exist') + cy.get('.attempt-1 .hook-item .collapsible:contains(test body)').should('not.exist') + cy.get('.attempt-1 .hook-item .collapsible:contains(after each)').should('not.exist') + cy.get('.attempt-1 .hook-item .collapsible:contains(after all)').should('not.exist') + + cy.contains('Attempt 2').click() + cy.get('.attempt-2 .hook-item .collapsible:contains(before each)') + cy.get('.attempt-2 .hook-item .collapsible:contains(before each (2))') + cy.get('.attempt-2 .hook-item .collapsible:contains(before each (3))').find('.command-state-failed') + cy.get('.attempt-2 .hook-item .collapsible:contains(test body)').should('not.exist') + cy.get('.attempt-2 .hook-item .collapsible:contains(after each)') + cy.get('.attempt-2 .hook-item .collapsible:contains(after all)').should('not.exist') + + cy.get('.attempt-3 .hook-item .collapsible:contains(before each)') + cy.get('.attempt-3 .hook-item .collapsible:contains(before each (2))') + cy.get('.attempt-3 .hook-item .collapsible:contains(before each (3))') + cy.get('.attempt-3 .hook-item .collapsible:contains(before each (4))') + cy.get('.attempt-3 .hook-item .collapsible:contains(test body)') + cy.get('.attempt-3 .hook-item .collapsible:contains(after each)') + cy.get('.attempt-3 .hook-item .collapsible:contains(after all)') + + cy.percySnapshot() + }) + + it('beforeEach retried tests skip remaining tests in suite', () => { + loadSpec({ + fileName: 'before-each-skip.retries.cy.js', + passCount: 0, + failCount: 1, + pendingCount: 0, + }) + + // ensure the page is loaded before taking snapshot + cy.contains('skips this') + + cy.percySnapshot() + }) + }) + + describe('afterEach', () => { + it('afterEach hooks retry on failure, but only run higher-level afterEach hooks', () => { + loadSpec({ + fileName: 'after-each-failure.retries.cy.js', + passCount: 1, + failCount: 0, + }) + + cy.contains('Attempt 1') + .click() + .then(shouldBeOpen) + + cy.get('.attempt-1 .hook-item .collapsible:contains(after each (1))').find('.command-state-failed') + cy.get('.attempt-1 .hook-item .collapsible:contains(after each (2))') + cy.get('.attempt-1 .hook-item .collapsible:contains(after each (3))').should('not.exist') + cy.get('.attempt-1 .hook-item .collapsible:contains(after all)').should('not.exist') + + cy.contains('Attempt 2').click() + .then(shouldBeOpen) + + cy.get('.attempt-2 .hook-item .collapsible:contains(after each (1))') + cy.get('.attempt-2 .hook-item .collapsible:contains(after each (2))') + cy.get('.attempt-2 .hook-item .collapsible:contains(after each (3))').find('.command-state-failed') + cy.get('.attempt-2 .hook-item .collapsible:contains(after all)').should('not.exist') + + cy.get('.attempt-tag').should('have.length', 3) + cy.get('.attempt-2 .hook-item .collapsible:contains(after each (1))') + cy.get('.attempt-2 .hook-item .collapsible:contains(after each (2))') + cy.get('.attempt-2 .hook-item .collapsible:contains(after each (3))') + cy.get('.attempt-3 .hook-item .collapsible:contains(after all)') + + cy.percySnapshot() + }) + + it('afterEach retried tests skip remaining tests in suite', () => { + loadSpec({ + fileName: 'after-each-skip.retries.cy.js', + passCount: 0, + failCount: 1, + pendingCount: 0, + }) + + cy.percySnapshot() + }) + }) + + describe('afterAll', () => { + it('only run afterAll hook on last attempt', () => { + loadSpec({ + fileName: 'after-all-once.retries.cy.js', + passCount: 3, + failCount: 0, + }) + + cy.contains('test 3').click() + getAttemptTag('test 3').first().click() + cy.contains('.attempt-1', 'after all').should('not.exist') + cy.contains('.attempt-2', 'after all') + }) + + it('tests do not retry when afterAll fails', () => { + loadSpec({ + fileName: 'after-all-failure.retries.cy.js', + passCount: 0, + failCount: 1, + }) + + cy.window().then((win) => { + cy.spy(win.console, 'error').as('console_error') + }) + + cy.contains('Although you have test retries') + cy.get('.runnable-err-print').click() + cy.get('@console_error').its('lastCall').should('be.calledWithMatch', 'Error') + + cy.percySnapshot() + }) + }) + + describe('can configure retries', () => { + after(() => { + window.top.__cySkipValidateConfig = false + }) + + const haveCorrectError = ($el) => cy.wrap($el).last().parentsUntil('.collapsible').last().parent().find('.runnable-err').should('contain', 'Unspecified AssertionError') + + window.top.__cySkipValidateConfig = true + + it('via config value', () => { + loadSpec({ + fileName: 'configure-retries.retries.cy.js', + passCount: 0, + failCount: 7, + }) + + getAttemptTag('[no retry]').should('have.length', 1).then(haveCorrectError) + getAttemptTag('[1 retry]').should('have.length', 2).then(haveCorrectError) + getAttemptTag('[2 retries]').should('have.length', 3).then(haveCorrectError) + getAttemptTag('[open mode, no retry]').should('have.length', 1).then(haveCorrectError) + getAttemptTag('[run mode, retry]').should('have.length', 2).then(haveCorrectError) + getAttemptTag('[open mode, 2 retries]').should('have.length', 3).then(haveCorrectError) + getAttemptTag('[set retries on suite]').should('have.length', 2).then(haveCorrectError) + }) + + it('throws when set via this.retries in test', () => { + loadSpec({ + fileName: 'configure-in-test.retries.cy.js', + passCount: 0, + failCount: 1, + }) + + cy.get('.runnable-err').should(containText(`it('tries to set mocha retries', { retries: 2 }, () => `)) + + cy.percySnapshot() + }) + + it('throws when set via this.retries in hook', () => { + loadSpec({ + fileName: 'configure-in-hook.retries.cy.js', + passCount: 0, + failCount: 1, + }) + + cy.get('.runnable-err').should(containText(`describe('suite 1', { retries: 0 }, () => `)) + + cy.percySnapshot() + }) + + it('throws when set via this.retries in suite', () => { + loadSpec({ + fileName: 'configure-in-suite.retries.cy.js', + passCount: 0, + failCount: 1, + }) + + cy.get('.runnable-err') + .should(containText(`describe('suite 1', { retries: 3 }, () => `)) + + cy.percySnapshot() + }) + }) +}) diff --git a/packages/app/cypress/e2e/runner/runner.ui.cy.ts b/packages/app/cypress/e2e/runner/runner.ui.cy.ts index aa33f5003e17..5c0888bdff2b 100644 --- a/packages/app/cypress/e2e/runner/runner.ui.cy.ts +++ b/packages/app/cypress/e2e/runner/runner.ui.cy.ts @@ -1,9 +1,9 @@ -import * as specLoader from './support/spec-loader' +import { loadSpec } from './support/spec-loader' -describe('src/cypress/runner', () => { +describe('runner ui', () => { describe('tests finish with correct state', () => { it('simple 1 test', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'simple-single-test.runner.cy.js', passCount: 1, failCount: 0, @@ -11,7 +11,7 @@ describe('src/cypress/runner', () => { }) it('simple 1 global test', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'simple-single-global-test.runner.cy.js', passCount: 1, failCount: 0, @@ -19,7 +19,7 @@ describe('src/cypress/runner', () => { }) it('simple 3 tests', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'three-simple-tests.runner.cy.js', passCount: 3, failCount: 0, @@ -27,7 +27,7 @@ describe('src/cypress/runner', () => { }) it('simple fail', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'simple-fail.runner.cy.js', passCount: 0, failCount: 1, @@ -38,7 +38,7 @@ describe('src/cypress/runner', () => { }) it('pass fail pass fail', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'pass-fail-pass-fail.runner.cy.js', passCount: 2, failCount: 2, @@ -46,7 +46,7 @@ describe('src/cypress/runner', () => { }) it('fail pass', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'fail-pass.runner.cy.js', passCount: 1, failCount: 1, @@ -54,7 +54,7 @@ describe('src/cypress/runner', () => { }) it('no tests', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'no-tests.runner.cy.js', passCount: 0, failCount: 0, @@ -65,7 +65,7 @@ describe('src/cypress/runner', () => { }) it('executes nested suite', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'nested-suite.runner.cy.js', passCount: 3, failCount: 0, @@ -73,7 +73,7 @@ describe('src/cypress/runner', () => { }) it('simple fail, catch cy.on(fail)', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'catch-fail.runner.cy.js', passCount: 1, failCount: 0, @@ -83,7 +83,7 @@ describe('src/cypress/runner', () => { describe('hook failures', () => { describe('test failures w/ hooks', () => { it('test [only]', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'test-only.runner.cy.js', passCount: 1, failCount: 0, @@ -91,7 +91,7 @@ describe('src/cypress/runner', () => { }) it('test [pending]', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'test-pending.runner.cy.js', passCount: 0, failCount: 0, @@ -100,7 +100,7 @@ describe('src/cypress/runner', () => { }) it('fail with [before]', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'fail-with-before.runner.cy.js', passCount: 1, failCount: 1, @@ -108,7 +108,7 @@ describe('src/cypress/runner', () => { }) it('fail with [after]', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'fail-with-after.runner.cy.js', passCount: 1, failCount: 1, @@ -116,7 +116,7 @@ describe('src/cypress/runner', () => { }) it('fail with all hooks', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'fail-with-all-hooks.runner.cy.js', passCount: 0, failCount: 1, @@ -128,7 +128,7 @@ describe('src/cypress/runner', () => { describe('other specs', () => { it('simple failing hook spec', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'failing-hooks.runner.cy.js', passCount: 1, failCount: 3, @@ -155,7 +155,7 @@ describe('src/cypress/runner', () => { }) it('async timeout spec', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'async-timeout.runner.cy.js', passCount: 0, failCount: 1, @@ -163,7 +163,7 @@ describe('src/cypress/runner', () => { }) it('scrolls each command into view', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'scrolls-command-log.runner.cy.js', passCount: 0, failCount: 1, @@ -173,7 +173,7 @@ describe('src/cypress/runner', () => { }) it('file with empty suites only displays no tests found', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'empty-suites.runner.cy.js', passCount: 0, failCount: 0, @@ -222,7 +222,7 @@ describe('src/cypress/runner', () => { describe('reporter interaction', () => { // https://github.com/cypress-io/cypress/issues/8621 it('user can stop test execution', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'stop-execution.runner.cy.js', passCount: 0, failCount: 1, @@ -233,7 +233,7 @@ describe('src/cypress/runner', () => { // TODO: determine intended function in new runner // it('supports disabling command log reporter with env var NO_COMMAND_LOG', () => { - // specLoader.loadSpec({ + // loadSpec({ // fileName: 'disabled-command-log.runner.cy.js', // passCount: 0, // failCount: 0, diff --git a/packages/app/cypress/e2e/runner/sessions.ui.cy.ts b/packages/app/cypress/e2e/runner/sessions.ui.cy.ts index 447edada7d89..ed21cf3fe06d 100644 --- a/packages/app/cypress/e2e/runner/sessions.ui.cy.ts +++ b/packages/app/cypress/e2e/runner/sessions.ui.cy.ts @@ -1,4 +1,4 @@ -import * as specLoader from './support/spec-loader' +import { loadSpec } from './support/spec-loader' describe('sessions ui', { viewportWidth: 1000, @@ -8,7 +8,7 @@ describe('sessions ui', { numTestsKeptInMemory: 1, }, () => { it('empty session with no data', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'blank_session.cy.js', passCount: 1, }) @@ -20,7 +20,7 @@ describe('sessions ui', { }) it('shows message for new, saved, and recreated session', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'recreated_session.cy.js', passCount: 3, }) @@ -52,7 +52,7 @@ describe('sessions ui', { }) it('multiple sessions in a test', () => { - specLoader.loadSpec({ + loadSpec({ fileName: 'multiple_sessions.cy.js', passCount: 1, }) diff --git a/packages/runner/cypress/e2e/retries.ui.cy.js b/packages/runner/cypress/e2e/retries.ui.cy.js deleted file mode 100644 index b7cb5802afbf..000000000000 --- a/packages/runner/cypress/e2e/retries.ui.cy.js +++ /dev/null @@ -1,497 +0,0 @@ -const helpers = require('../support/helpers') - -const { shouldHaveTestResults, containText } = helpers -const { runIsolatedCypress } = helpers.createCypress({ config: { retries: 2 } }) - -const getAttemptTag = (sel) => { - return cy.get(`.test.runnable:contains(${sel}) .attempt-tag`) -} - -const shouldBeOpen = ($el) => cy.wrap($el).parentsUntil('.collapsible').last().parent().should('have.class', 'is-open') - -const attemptTag = (sel) => `.attempt-tag:contains(Attempt ${sel})` -const cyReject = (fn) => { - return () => { - try { - fn() - } catch (e) { - cy.state('reject')(e) - } - } -} - -describe('runner/cypress retries.ui.spec', { viewportWidth: 600, viewportHeight: 900 }, () => { - it('collapses tests that retry and pass', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { - tests: [ - { name: 'test pass', fail: 0 }, - { name: 'test pass on 2nd attempt', fail: 1 }, - ], - }, - }, - }) - .then(shouldHaveTestResults(2, 0)) - - cy.get('.test').should('have.length', 2) - cy.percySnapshot() - }) - - it('collapses prev attempts and keeps final one open on failure', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { - tests: [ - { name: 'test 1', - fail: true, - }, - { name: 'test 2', - - }, - ], - }, - }, - }, { config: { retries: 2 } }) - .then(shouldHaveTestResults(1, 1)) - - cy.get('.runnable-err-print').should('be.visible') - cy.percySnapshot() - }) - - it('can toggle failed prev attempt open and log its error', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { - tests: [ - { name: 'test 1', fail: 1 }, - { name: 'test 2', fail: 2 }, - { name: 'test 3', fail: 1 }, - ], - }, - }, - }, { config: { retries: 1 } }) - .then(shouldHaveTestResults(2, 1)) - .then(() => { - cy.contains('Attempt 1') - .click() - .closest('.attempt-item') - .find('.runnable-err-print') - .click() - - cy.get('@console_error').should('be.calledWithMatch', 'AssertionError: test 2') - }) - - cy.percySnapshot() - }) - - it('opens attempt on each attempt failure for the screenshot, and closes after test passes', { retries: 2 }, () => { - let stub - - runIsolatedCypress( - { - suites: { - 's1': { - tests: [ - 't1', - { - name: 't2', - fail: 3, - }, - 't3', - ], - }, - }, - }, { config: { retries: 3, isTextTerminal: true }, - onBeforeRun ({ autCypress }) { - let attempt = 0 - - stub = cy.stub().callsFake(cyReject(() => { - attempt++ - expect(cy.$$('.attempt-item > .is-open').length).to.equal(attempt) - })) - - autCypress.Screenshot.onAfterScreenshot = stub - }, - }, - ).then(() => { - expect(stub).callCount(3) - cy.get('.test.runnable:contains(t2)').then(($el) => { - expect($el).not.class('is-open') - }) - }) - }) - - it('includes routes, spies, hooks, and commands in attempt', () => { - runIsolatedCypress({ - suites: { - 's1': { - hooks: [{ type: 'beforeEach', fail: 1, agents: true }], - tests: [{ name: 't1', fail: 1, agents: true }], - }, - }, - }) - .then(shouldHaveTestResults(1, 0)) - .then(() => { - cy.get(attemptTag(1)).click().parentsUntil('.collapsible').last().parent().within(() => { - cy.get('.instruments-container').should('contain', 'Spies / Stubs (1)') - cy.get('.instruments-container').should('contain', 'Routes (1)') - cy.get('.runnable-err').should('contain', 'AssertionError') - }) - - cy.get(attemptTag(2)).click().parentsUntil('.collapsible').last().parent().within(() => { - cy.get('.instruments-container').should('contain', 'Spies / Stubs (2)') - cy.get('.instruments-container').should('contain', 'Routes (2)') - cy.get('.runnable-err').should('contain', 'AssertionError') - }) - - cy.get(attemptTag(3)).parentsUntil('.collapsible').last().parent().within(() => { - cy.get('.instruments-container').should('contain', 'Spies / Stubs (2)') - cy.get('.instruments-container').should('contain', 'Routes (2)') - cy.get('.runnable-err').should('not.be.visible') - }) - }) - - cy.percySnapshot() - }) - - describe('only', () => { - it('test retry with [only]', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { - hooks: ['before', 'beforeEach', 'afterEach', 'after'], - tests: [ - { name: 'test 1' }, - { name: 'test 2', fail: 1, only: true }, - { name: 'test 3' }, - ], - }, - }, - }, { config: { retries: 1 } }) - .then(shouldHaveTestResults(1, 0)) - .then(() => { - cy.contains('test 2') - cy.contains('test 1').should('not.exist') - cy.contains('test 3').should('not.exist') - }) - - cy.percySnapshot() - }) - }) - - describe('beforeAll', () => { - // TODO: make beforeAll hooks retry - it('tests do not retry when beforeAll fails', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { - hooks: [ - { type: 'before', fail: 1 }, - 'beforeEach', - 'beforeEach', - 'afterEach', - 'afterEach', - 'after', - ], - tests: ['test 1'], - }, - }, - }, { config: { retries: 1 } }) - .then(shouldHaveTestResults(0, 1)) - .then(() => { - cy.contains('Although you have test retries') - }) - - cy.percySnapshot() - }) - - // TODO: future versions should run all hooks associated with test on retry - it('before all hooks are not run on the second attempt when fails outside of beforeAll', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { - hooks: ['before', 'beforeEach', 'afterEach', 'after'], - tests: [{ name: 'test 1', fail: 1 }], - }, - }, - }, { config: { retries: 1 } }) - .then(shouldHaveTestResults(1, 0)) - .then(() => { - cy.contains('test') - cy.contains('after all') - cy.contains('before all').should('not.exist') - }) - - cy.percySnapshot() - }) - }) - - describe('beforeEach', () => { - it('beforeEach hooks retry on failure, but only run same-level afterEach hooks', () => { - runIsolatedCypress({ - hooks: [{ type: 'beforeEach', fail: 1 }], - suites: { - 'suite 1': { - hooks: [ - 'before', - 'beforeEach', - { type: 'beforeEach', fail: 1 }, - 'beforeEach', - 'afterEach', - 'after', - ], - tests: [{ name: 'test 1' }], - }, - }, - }, { config: { retries: 2 } }) - .then(shouldHaveTestResults(1, 0)) - .then(() => { - cy.contains('Attempt 1').click() - cy.get('.attempt-1 .hook-item .collapsible:contains(before each)').find('.command-state-failed') - cy.get('.attempt-1 .hook-item .collapsible:contains(before each (2))').should('not.exist') - cy.get('.attempt-1 .hook-item .collapsible:contains(test body)').should('not.exist') - cy.get('.attempt-1 .hook-item .collapsible:contains(after each)').should('not.exist') - cy.get('.attempt-1 .hook-item .collapsible:contains(after all)').should('not.exist') - - cy.contains('Attempt 2').click() - cy.get('.attempt-2 .hook-item .collapsible:contains(before each)') - cy.get('.attempt-2 .hook-item .collapsible:contains(before each (2))') - cy.get('.attempt-2 .hook-item .collapsible:contains(before each (3))').find('.command-state-failed') - cy.get('.attempt-2 .hook-item .collapsible:contains(test body)').should('not.exist') - cy.get('.attempt-2 .hook-item .collapsible:contains(after each)') - cy.get('.attempt-2 .hook-item .collapsible:contains(after all)').should('not.exist') - - cy.get('.attempt-3 .hook-item .collapsible:contains(before each)') - cy.get('.attempt-3 .hook-item .collapsible:contains(before each (2))') - cy.get('.attempt-3 .hook-item .collapsible:contains(before each (3))') - cy.get('.attempt-3 .hook-item .collapsible:contains(before each (4))') - cy.get('.attempt-3 .hook-item .collapsible:contains(test body)') - cy.get('.attempt-3 .hook-item .collapsible:contains(after each)') - cy.get('.attempt-3 .hook-item .collapsible:contains(after all)') - }) - - cy.percySnapshot() - }) - - it('beforeEach retried tests skip remaining tests in suite', () => { - runIsolatedCypress({ suites: { - 'beforeEach hooks': { - hooks: [{ type: 'beforeEach', fail: true }], - tests: ['fails in beforeEach', 'skips this'], - }, - - } }, { config: { retries: 1 } }) - .then(shouldHaveTestResults(0, 1, 0)) - - // ensure the page is loaded before taking snapshot - cy.contains('skips this') - cy.percySnapshot() - }) - }) - - describe('afterEach', () => { - it('afterEach hooks retry on failure, but only run higher-level afterEach hooks', () => { - runIsolatedCypress({ - hooks: [{ type: 'afterEach', fail: 2 }], - suites: { - 's1': { - hooks: [{ type: 'afterEach', fail: 1 }, 'afterEach', 'after'], - tests: ['t1'], - }, - - }, - }, { config: { retries: 2 } }) - .then(shouldHaveTestResults(1, 0)) - .then(() => { - cy.contains('Attempt 1') - .click() - .then(shouldBeOpen) - - cy.get('.attempt-1 .hook-item .collapsible:contains(after each (1))').find('.command-state-failed') - cy.get('.attempt-1 .hook-item .collapsible:contains(after each (2))') - cy.get('.attempt-1 .hook-item .collapsible:contains(after each (3))').should('not.exist') - cy.get('.attempt-1 .hook-item .collapsible:contains(after all)').should('not.exist') - - cy.contains('Attempt 2').click() - .then(shouldBeOpen) - - cy.get('.attempt-2 .hook-item .collapsible:contains(after each (1))') - cy.get('.attempt-2 .hook-item .collapsible:contains(after each (2))') - cy.get('.attempt-2 .hook-item .collapsible:contains(after each (3))').find('.command-state-failed') - cy.get('.attempt-2 .hook-item .collapsible:contains(after all)').should('not.exist') - - cy.get('.attempt-tag').should('have.length', 3) - cy.get('.attempt-2 .hook-item .collapsible:contains(after each (1))') - cy.get('.attempt-2 .hook-item .collapsible:contains(after each (2))') - cy.get('.attempt-2 .hook-item .collapsible:contains(after each (3))') - cy.get('.attempt-3 .hook-item .collapsible:contains(after all)') - }) - - cy.percySnapshot() - }) - - it('afterEach retried tests skip remaining tests in suite', () => { - runIsolatedCypress({ suites: { - 'afterEach hooks': { - hooks: [{ type: 'afterEach', fail: true }], - tests: ['fails in afterEach', 'skips this'], - }, - - } }, { config: { retries: 1 } }) - .then(shouldHaveTestResults(0, 1, 0)) - - cy.percySnapshot() - }) - }) - - describe('afterAll', () => { - it('only run afterAll hook on last attempt', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { - hooks: [ - 'before', - 'beforeEach', - 'afterEach', - 'after', - ], - tests: [ - { name: 'test 1' }, - { name: 'test 2' }, - { name: 'test 3', fail: 1 }, - ], - }, - }, - }, { config: { retries: 1 } }) - .then(shouldHaveTestResults(3, 0)) - .then(() => { - cy.contains('test 3').click() - getAttemptTag('test 3').first().click() - cy.contains('.attempt-1', 'after all').should('not.exist') - cy.contains('.attempt-2', 'after all') - }) - }) - - it('tests do not retry when afterAll fails', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { - hooks: [ - 'before', - 'beforeEach', - 'beforeEach', - 'afterEach', - 'afterEach', - { type: 'after', fail: 1 }, - ], - tests: [{ name: 'test 1' }], - }, - }, - }, { config: { retries: 1 } }) - .then(shouldHaveTestResults(0, 1)) - .then(() => { - cy.contains('Although you have test retries') - cy.get('.runnable-err-print').click() - cy.get('@console_error').its('lastCall').should('be.calledWithMatch', 'Error') - }) - - cy.percySnapshot() - }) - }) - - describe('can configure retries', () => { - after(() => { - window.top.__cySkipValidateConfig = false - }) - - const haveCorrectError = ($el) => cy.wrap($el).last().parentsUntil('.collapsible').last().parent().find('.runnable-err').should('contain', 'Unspecified AssertionError') - - window.top.__cySkipValidateConfig = true - it('via config value', () => { - runIsolatedCypress({ - suites: { - 'suite 1': () => { - it('[no retry]', { retries: 0 }, () => assert(false)) - it('[1 retry]', { retries: 1 }, () => assert(false)) - it('[2 retries]', { retries: 2 }, () => assert(false)) - it('[open mode, no retry]', { retries: { runMode: 2, openMode: 0 } }, () => assert(false)) - it('[run mode, retry]', { retries: { runMode: 1, openMode: 0 }, isInteractive: false }, () => assert(false)) - it('[open mode, 2 retries]', { isInteractive: true }, () => assert(false)) - describe('suite 2', { retries: 1 }, () => { - it('[set retries on suite]', () => assert(false)) - }) - }, - }, - }) - .then(shouldHaveTestResults(0, 7)) - .then(() => { - getAttemptTag('[no retry]').should('have.length', 1).then(haveCorrectError) - getAttemptTag('[1 retry]').should('have.length', 2).then(haveCorrectError) - getAttemptTag('[2 retries]').should('have.length', 3).then(haveCorrectError) - getAttemptTag('[open mode, no retry]').should('have.length', 1).then(haveCorrectError) - getAttemptTag('[run mode, retry]').should('have.length', 2).then(haveCorrectError) - getAttemptTag('[open mode, 2 retries]').should('have.length', 3).then(haveCorrectError) - getAttemptTag('[set retries on suite]').should('have.length', 2).then(haveCorrectError) - }) - }) - - it('throws when set via this.retries in test', () => { - runIsolatedCypress({ - suites: { - 'suite 1': () => { - it('tries to set mocha retries', function () { - this.retries(null) - }) - }, - }, - }) - .then(shouldHaveTestResults(0, 1)) - .then(() => { - cy.get('.runnable-err').should(containText(`it('tries to set mocha retries', { retries: 2 }, () => `)) - }) - - cy.percySnapshot() - }) - - it('throws when set via this.retries in hook', () => { - runIsolatedCypress({ - suites: { - 'suite 1': () => { - beforeEach(function () { - this.retries(0) - }) - - it('foo', () => {}) - }, - }, - }) - .then(shouldHaveTestResults(0, 1)) - .then(() => { - cy.get('.runnable-err').should(containText(`describe('suite 1', { retries: 0 }, () => `)) - }) - - cy.percySnapshot() - }) - - it('throws when set via this.retries in suite', () => { - runIsolatedCypress({ - suites: { - // eslint-disable-next-line object-shorthand - 'suite 1': function () { - this.retries(3) - it('test 1', () => { - }) - }, - }, - }) - .then(shouldHaveTestResults(0, 1)) - .then(() => { - cy.get('.runnable-err') - .should(containText(`describe('suite 1', { retries: 3 }, () => `)) - }) - - cy.percySnapshot() - }) - }) -}) diff --git a/packages/runner/cypress/e2e/runner.ui.cy.js b/packages/runner/cypress/e2e/runner.ui.cy.js deleted file mode 100644 index 00dd7253e35b..000000000000 --- a/packages/runner/cypress/e2e/runner.ui.cy.js +++ /dev/null @@ -1,408 +0,0 @@ -const helpers = require('../support/helpers') - -const { shouldHaveTestResults, createCypress } = helpers -const { runIsolatedCypress } = createCypress() - -const fx_simpleSingleTest = { - suites: { 'suite 1': { tests: [{ name: 'test 1' }] } }, -} - -const fx_failPass = { - suites: { - 'suite 1': { - tests: [ - { - name: 'test 1', - fail: true, - }, - { name: 'test 2' }, - ], - }, - }, -} - -const fx_passFailPassFail = { - suites: { - 'suite 1': { - tests: [ - 'test 1', - { - name: 'test 2', - fail: true, - }, - ], - }, - 'suite 2': { - tests: [ - 'test 1', - { - name: 'test 2', - fail: true, - }, - ], - }, - }, -} - -describe('src/cypress/runner', () => { - describe('tests finish with correct state', () => { - it('simple 1 test', () => { - runIsolatedCypress(fx_simpleSingleTest) - .then(shouldHaveTestResults(1, 0)) - }) - - it('simple 1 global test', () => { - runIsolatedCypress(() => { - it('foo', () => { - expect(true).is.true - }) - }) - .then(shouldHaveTestResults(1, 0)) - }) - - it('simple 3 tests', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { tests: ['test 1', 'test 2', 'test 3'] }, - }, - }) - .then(shouldHaveTestResults(3, 0)) - }) - - it('simple fail', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { - tests: [ - { - name: 'test 1', - fail: true, - }, - ], - }, - }, - }) - .then(shouldHaveTestResults(0, 1)) - .then(() => { - // render exactly one error - cy.get('.runnable-err:contains(AssertionError)').should('have.length', 1) - }) - }) - - it('pass fail pass fail', () => { - runIsolatedCypress(fx_passFailPassFail) - .then(shouldHaveTestResults(2, 2)) - }) - - it('fail pass', () => { - runIsolatedCypress(fx_failPass) - .then(shouldHaveTestResults(1, 1)) - }) - - it('no tests', () => { - runIsolatedCypress({}) - .then(shouldHaveTestResults(0, 0)) - - cy.contains('No tests found.').should('be.visible') - cy.contains('p', 'Cypress could not detect tests in this file.').should('be.visible') - }) - - it('ends test before nested suite', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { tests: ['test 1', 'test 2'], - suites: { - 'suite 1-1': { - tests: ['test 1'], - }, - } }, - }, - }, {}) - .then(shouldHaveTestResults(3, 0)) - }) - - it('simple fail, catch cy.on(fail)', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { - tests: [ - { - name: 'test 1', - fn: () => { - cy.on('fail', () => { - return false - }) - - expect(false).ok - throw new Error('error in test') - }, - eval: true, - }, - ], - }, - }, - }) - .then(shouldHaveTestResults(1, 0)) - }) - - describe('hook failures', () => { - describe('test failures w/ hooks', () => { - it('test [only]', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { - hooks: ['before', 'beforeEach', 'afterEach', 'after'], - tests: [ - { name: 'test 1' }, - { name: 'test 2', only: true }, - { name: 'test 3' }, - ], - }, - }, - }).then(shouldHaveTestResults(1, 0)) - }) - - it('test [pending]', () => { - runIsolatedCypress(() => { - before(() => {}) - it('t1') - it('t2') - it('t3') - after(() => {}) - }).then(shouldHaveTestResults(0, 0, 3)) - }) - - it('fail with [before]', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { - hooks: ['before'], - tests: [ - { - name: 'test 1', - fail: true, - }, - { name: 'test 2' }, - ], - }, - }, - }) - .then(shouldHaveTestResults(1, 1)) - }) - - it('fail with [after]', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { - hooks: [{ type: 'after' }], - tests: [{ name: 'test 1', fail: true }, 'test 2'], - }, - }, - }) - .then(shouldHaveTestResults(1, 1)) - }) - - it('fail with all hooks', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { - hooks: ['before', 'beforeEach', 'afterEach', 'after'], - tests: [{ name: 'test 1', fail: true }], - }, - }, - }) - .then(shouldHaveTestResults(0, 1)) - }) - }) - }) - }) - - describe('other specs', () => { - it('simple failing hook spec', () => { - const mochaTests = { - suites: { - 'simple failing hook spec': { - suites: { - 'beforeEach hooks': { - hooks: [{ type: 'beforeEach', fail: true }], - tests: ['never gets here'], - }, - 'pending': { - tests: [{ name: 'is pending', pending: true }], - }, - 'afterEach hooks': { - hooks: [{ type: 'afterEach', fail: true }], - tests: ['fails this', 'does not run this'], - }, - 'after hooks': { - hooks: [{ type: 'after', fail: true }] - , tests: ['runs this', 'fails on this'], - }, - }, - }, - - }, - } - - runIsolatedCypress(mochaTests) - .then(shouldHaveTestResults(1, 3)) - .then(() => { - cy.contains('.test', 'never gets here').should('have.class', 'runnable-failed') - cy.contains('.command', 'beforeEach').should('have.class', 'command-state-failed') - cy.contains('.runnable-err', 'beforeEach').scrollIntoView().should('be.visible') - - cy.contains('.test', 'is pending').should('have.class', 'runnable-pending') - - cy.contains('.test', 'fails this').should('have.class', 'runnable-failed') - cy.contains('.command', 'afterEach').should('have.class', 'command-state-failed') - cy.contains('.runnable-err', 'afterEach').should('be.visible') - - cy.contains('.test', 'does not run this').should('have.class', 'runnable-processing') - - cy.contains('.test', 'runs this').should('have.class', 'runnable-passed') - - cy.contains('.test', 'fails on this').should('have.class', 'runnable-failed') - cy.contains('.command', 'after').should('have.class', 'command-state-failed') - cy.contains('.runnable-err', 'after').should('be.visible') - }) - }) - - it('async timeout spec', () => { - runIsolatedCypress({ - suites: { - 'async': { - tests: [ - { name: 'bar fails', - // eslint-disable-next-line - fn (done) { - this.timeout(100) - cy.on('fail', () => {}) - // eslint-disable-next-line - foo.bar() - }, - eval: true, - }, - ], - }, - }, - }) - .then(shouldHaveTestResults(0, 1)) - }) - - it('scrolls each command into view', () => { - // HACK to assert on the dom DURING the runIsolatedCypress run - // we expect the last command item to be scrolled into view before - // the test ends - const result = cy.now('get', '.command-number:contains(25):visible').catch((e) => cy.state('reject')(e)) - - runIsolatedCypress(() => { - describe('s1', () => { - // Passing in done forces the spec to timeout - // eslint-disable-next-line - it('t1', (done) => { - cy.timeout(10) - Cypress._.times(25, () => expect(true).ok) - }) - }) - }) - - cy.wrap(result) - }) - - it('file with empty suites only displays no tests found', () => { - runIsolatedCypress({ - suites: { - 'suite 1': { - suites: { - 'suite 2': {}, - }, - }, - }, - }).then(() => { - cy.get('.reporter').contains('No tests found') - }) - }) - }) - - describe('runner header', () => { - context('viewport dropdown', () => { - it('shows on click', () => { - runIsolatedCypress({}) - cy.get('.viewport-menu').should('not.be.visible') - cy.get('.viewport-info button').click() - cy.get('.viewport-menu').should('be.visible') - - cy.percySnapshot() - }) - }) - - context('selector playground', () => { - it('shows on click', () => { - runIsolatedCypress({}) - - cy.get('.selector-playground').should('not.be.visible') - cy.get('.selector-playground-toggle').click() - cy.get('.selector-playground').should('be.visible') - - cy.percySnapshot() - }) - - it('closes on restart', () => { - runIsolatedCypress({}) - - cy.get('.selector-playground-toggle').click() - cy.get('.selector-playground').should('be.visible') - - cy.get('.restart').click() - cy.get('.selector-playground').should('not.be.visible') - }) - }) - }) - - describe('reporter interaction', () => { - // https://github.com/cypress-io/cypress/issues/8621 - it('user can stop test execution', (done) => { - runIsolatedCypress(() => { - // eslint-disable-next-line mocha/handle-done-callback - it('test stops while running', (done) => { - cy.timeout(200) - cy.get('.not-exist') - setTimeout(() => { - cy.$$('button.stop', parent.document).click() - }, 100) - }) - - afterEach(function () { - this.currentTest.err = new Error('ran aftereach') - }) - }, { - onBeforeRun ({ autCypress }) { - autCypress.on('test:after:run', (arg) => { - expect(arg.err.message).not.contain('aftereach') - done() - }) - }, - }) - }) - - it('supports disabling command log reporter with env var NO_COMMAND_LOG', () => { - runIsolatedCypress(() => { - it('foo', () => { - // simulate a page load, ensures reporter state event is properly stubbed - cy.then(() => Cypress.action('cy:collect:run:state')) - cy.visit('/') - - // ensures runner doesn't wait for nonexist before:screenshot ack - cy.screenshot({ - capture: 'runner', - }) - }) - }, - { - config: { env: { NO_COMMAND_LOG: '1' } }, - }) - - cy.get('.reporter').should('not.exist') - }) - }) -}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/after-all-failure.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/after-all-failure.retries.cy.js new file mode 100644 index 000000000000..45bdba292fbb --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/after-all-failure.retries.cy.js @@ -0,0 +1,18 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + options: { retries: 1 }, + hooks: [ + 'before', + 'beforeEach', + 'beforeEach', + 'afterEach', + 'afterEach', + { type: 'after', fail: 1 }, + ], + tests: [{ name: 'test 1' }], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/after-all-once.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/after-all-once.retries.cy.js new file mode 100644 index 000000000000..3a2a00806789 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/after-all-once.retries.cy.js @@ -0,0 +1,20 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + options: { retries: 1 }, + hooks: [ + 'before', + 'beforeEach', + 'afterEach', + 'after', + ], + tests: [ + { name: 'test 1' }, + { name: 'test 2' }, + { name: 'test 3', fail: 1 }, + ], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/after-each-failure.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/after-each-failure.retries.cy.js new file mode 100644 index 000000000000..c6564767510b --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/after-each-failure.retries.cy.js @@ -0,0 +1,14 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + hooks: [{ type: 'afterEach', fail: 2 }], + suites: { + 'suite 1': { + options: { + retries: 2, + }, + hooks: [{ type: 'afterEach', fail: 1 }, 'afterEach', 'after'], + tests: ['t1'], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/after-each-skip.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/after-each-skip.retries.cy.js new file mode 100644 index 000000000000..e5c52e66662a --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/after-each-skip.retries.cy.js @@ -0,0 +1,13 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'afterEach hooks': { + options: { + retries: 1, + }, + hooks: [{ type: 'afterEach', fail: true }], + tests: ['fails in afterEach', 'skips this'], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/all-retry-one-failure.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/all-retry-one-failure.retries.cy.js new file mode 100644 index 000000000000..f0add6075138 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/all-retry-one-failure.retries.cy.js @@ -0,0 +1,16 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + options: { + retries: 1, + }, + tests: [ + { name: 'test 1', fail: 1 }, + { name: 'test 2', fail: 2 }, + { name: 'test 3', fail: 1 }, + ], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/before-all-called-once.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/before-all-called-once.retries.cy.js new file mode 100644 index 000000000000..a41f8b68508d --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/before-all-called-once.retries.cy.js @@ -0,0 +1,13 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + options: { + retries: 1, + }, + hooks: ['before', 'beforeEach', 'afterEach', 'after'], + tests: [{ name: 'test 1', fail: 1 }], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/before-all-failure.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/before-all-failure.retries.cy.js new file mode 100644 index 000000000000..505e74d62a6e --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/before-all-failure.retries.cy.js @@ -0,0 +1,20 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + options: { + retries: 1, + }, + hooks: [ + { type: 'before', fail: 1 }, + 'beforeEach', + 'beforeEach', + 'afterEach', + 'afterEach', + 'after', + ], + tests: ['test 1'], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/before-each-failure.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/before-each-failure.retries.cy.js new file mode 100644 index 000000000000..7cac2ab4e50d --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/before-each-failure.retries.cy.js @@ -0,0 +1,21 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + hooks: [{ type: 'beforeEach', fail: 1 }], + suites: { + 'suite 1': { + options: { + retries: 2, + }, + hooks: [ + 'before', + 'beforeEach', + { type: 'beforeEach', fail: 1 }, + 'beforeEach', + 'afterEach', + 'after', + ], + tests: [{ name: 'test 1' }], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/before-each-skip.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/before-each-skip.retries.cy.js new file mode 100644 index 000000000000..0f3915355c20 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/before-each-skip.retries.cy.js @@ -0,0 +1,13 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'beforeEach hooks': { + options: { + retries: 1, + }, + hooks: [{ type: 'beforeEach', fail: true }], + tests: ['fails in beforeEach', 'skips this'], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/collapse-after-pass.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/collapse-after-pass.retries.cy.js new file mode 100644 index 000000000000..cd1675552dd7 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/collapse-after-pass.retries.cy.js @@ -0,0 +1,15 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + options: { + retries: 2, + }, + tests: [ + { name: 'test pass', fail: 0 }, + { name: 'test pass on 2nd attempt', fail: 1 }, + ], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/collapse-prev-attempts.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/collapse-prev-attempts.retries.cy.js new file mode 100644 index 000000000000..93bc5a57e7f1 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/collapse-prev-attempts.retries.cy.js @@ -0,0 +1,20 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + options: { + retries: 2, + }, + tests: [ + { + name: 'test 1', + fail: true, + }, + { + name: 'test 2', + }, + ], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/configure-in-hook.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/configure-in-hook.retries.cy.js new file mode 100644 index 000000000000..5537fcbc0d6e --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/configure-in-hook.retries.cy.js @@ -0,0 +1,14 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': () => { + beforeEach(function () { + this.retries(0) + }) + + it('foo', () => {}) + }, + }, + +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/configure-in-suite.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/configure-in-suite.retries.cy.js new file mode 100644 index 000000000000..05b6e9213239 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/configure-in-suite.retries.cy.js @@ -0,0 +1,12 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + // eslint-disable-next-line object-shorthand + 'suite 1': function () { + this.retries(3) + it('test 1', () => { + }) + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/configure-in-test.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/configure-in-test.retries.cy.js new file mode 100644 index 000000000000..13a2a04e8617 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/configure-in-test.retries.cy.js @@ -0,0 +1,11 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': () => { + it('tries to set mocha retries', function () { + this.retries(null) + }) + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/configure-retries.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/configure-retries.retries.cy.js new file mode 100644 index 000000000000..bddaf29eb46f --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/configure-retries.retries.cy.js @@ -0,0 +1,17 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': () => { + it('[no retry]', { retries: 0 }, () => assert(false)) + it('[1 retry]', { retries: 1 }, () => assert(false)) + it('[2 retries]', { retries: 2 }, () => assert(false)) + it('[open mode, no retry]', { retries: { runMode: 2, openMode: 0 } }, () => assert(false)) + it('[run mode, retry]', { retries: { runMode: 1, openMode: 0 }, isInteractive: false }, () => assert(false)) + it('[open mode, 2 retries]', { retries: { runMode: 0, openMode: 2 }, isInteractive: true }, () => assert(false)) + describe('suite 2', { retries: 1 }, () => { + it('[set retries on suite]', () => assert(false)) + }) + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/includes-all-in-attempt.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/includes-all-in-attempt.retries.cy.js new file mode 100644 index 000000000000..3556c6f3fb12 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/includes-all-in-attempt.retries.cy.js @@ -0,0 +1,13 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 's1': { + options: { + retries: 2, + }, + hooks: [{ type: 'beforeEach', fail: 1, agents: true }], + tests: [{ name: 't1', fail: 1, agents: true }], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/only.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/only.retries.cy.js new file mode 100644 index 000000000000..d0910c35c4e2 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/only.retries.cy.js @@ -0,0 +1,17 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +generateMochaTestsForWin(window, { + suites: { + 'suite 1': { + options: { + retries: 2, + }, + hooks: ['before', 'beforeEach', 'afterEach', 'after'], + tests: [ + { name: 'test 1' }, + { name: 'test 2', fail: 1, only: true }, + { name: 'test 3' }, + ], + }, + }, +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/opens-attempt-for-screenshot.retries.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/opens-attempt-for-screenshot.retries.cy.js new file mode 100644 index 000000000000..bee5d9eb938d --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/retries/opens-attempt-for-screenshot.retries.cy.js @@ -0,0 +1,25 @@ +import { generateMochaTestsForWin } from '../support/generate-mocha-tests' + +before(() => { + window.__cySkipValidateConfig = true +}) + +generateMochaTestsForWin(window, { + suites: { + 's1': { + options: { retries: 3, isTextTerminal: true }, + tests: [ + 't1', + { + name: 't2', + fail: 3, + }, + 't3', + ], + }, + }, +}) + +after(() => { + window.__cySkipValidateConfig = false +}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/support/generate-mocha-tests.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/support/generate-mocha-tests.js index 216196b85b98..4bb295c651a3 100644 --- a/system-tests/projects/runner-e2e-specs/cypress/e2e/support/generate-mocha-tests.js +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/support/generate-mocha-tests.js @@ -152,13 +152,13 @@ export const createSuites = (win, suites = {}) => { fn = evalFn(win, obj) } - win.describe(suiteName, fn) + win.describe(suiteName, obj.options || {}, fn) }) } export const generateMochaTestsForWin = (win, obj) => { if (typeof obj === 'function') { - win.eval(`( ${obj.toString()})()`) + win.eval(`(${obj.toString()})()`) return } From cb0305755e64b4b1d91cdfa23d1b19129af86b0d Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Wed, 9 Feb 2022 10:16:25 -0600 Subject: [PATCH 24/64] Cleaning up a few things for consistency --- .../cypress/e2e/runner/reporter.hooks.cy.ts | 178 ++++++++---------- .../app/cypress/e2e/runner/retries.ui.cy.ts | 115 +++++------ 2 files changed, 142 insertions(+), 151 deletions(-) diff --git a/packages/app/cypress/e2e/runner/reporter.hooks.cy.ts b/packages/app/cypress/e2e/runner/reporter.hooks.cy.ts index 198fc4885b7b..e800536d040c 100644 --- a/packages/app/cypress/e2e/runner/reporter.hooks.cy.ts +++ b/packages/app/cypress/e2e/runner/reporter.hooks.cy.ts @@ -5,129 +5,111 @@ describe('hooks', { // of nested spec snapshots numTestsKeptInMemory: 1, }, () => { - describe('displays hooks', () => { - beforeEach(() => { - loadSpec({ - fileName: 'basic.cy.js', - passCount: 2, - }) + it('displays commands under correct hook', () => { + loadSpec({ + fileName: 'basic.cy.js', + passCount: 2, }) - it('displays commands under correct hook', () => { - cy.contains('tests 1').click() - - cy.contains('before all').closest('.collapsible').should('contain', 'beforeHook 1') - cy.contains('before each').closest('.collapsible').should('contain', 'beforeEachHook 1') - cy.contains('test body').closest('.collapsible').should('contain', 'testBody 1') - cy.contains('after each').closest('.collapsible').should('contain', 'afterEachHook 1') - - // displays hooks without number when only one of type - cy.contains('before all').should('not.contain', '(1)') - cy.contains('before each').should('not.contain', '(1)') - cy.contains('after each').should('not.contain', '(1)') - - // displays hooks separately with number when more than one of type - cy.contains('tests 1').click() - cy.contains('tests 2').click() - - cy.contains('before all (1)').closest('.collapsible').should('contain', 'beforeHook 2') - cy.contains('before all (2)').closest('.collapsible').should('contain', 'beforeHook 3') - cy.contains('before each (1)').closest('.collapsible').should('contain', 'beforeEachHook 1') - cy.contains('before each (2)').closest('.collapsible').should('contain', 'beforeEachHook 2') - cy.contains('test body').closest('.collapsible').should('contain', 'testBody 2') - cy.contains('after each (1)').closest('.collapsible').should('contain', 'afterEachHook 2') - cy.contains('after each (2)').closest('.collapsible').should('contain', 'afterEachHook 1') - cy.contains('after all (1)').closest('.collapsible').should('contain', 'afterHook 2') - cy.contains('after all (2)').closest('.collapsible').should('contain', 'afterHook 1') - }) + cy.contains('tests 1').click() + + cy.contains('before all').closest('.collapsible').should('contain', 'beforeHook 1') + cy.contains('before each').closest('.collapsible').should('contain', 'beforeEachHook 1') + cy.contains('test body').closest('.collapsible').should('contain', 'testBody 1') + cy.contains('after each').closest('.collapsible').should('contain', 'afterEachHook 1') + + // displays hooks without number when only one of type + cy.contains('before all').should('not.contain', '(1)') + cy.contains('before each').should('not.contain', '(1)') + cy.contains('after each').should('not.contain', '(1)') + + // displays hooks separately with number when more than one of type + cy.contains('tests 1').click() + cy.contains('tests 2').click() + + cy.contains('before all (1)').closest('.collapsible').should('contain', 'beforeHook 2') + cy.contains('before all (2)').closest('.collapsible').should('contain', 'beforeHook 3') + cy.contains('before each (1)').closest('.collapsible').should('contain', 'beforeEachHook 1') + cy.contains('before each (2)').closest('.collapsible').should('contain', 'beforeEachHook 2') + cy.contains('test body').closest('.collapsible').should('contain', 'testBody 2') + cy.contains('after each (1)').closest('.collapsible').should('contain', 'afterEachHook 2') + cy.contains('after each (2)').closest('.collapsible').should('contain', 'afterEachHook 1') + cy.contains('after all (1)').closest('.collapsible').should('contain', 'afterHook 2') + cy.contains('after all (2)').closest('.collapsible').should('contain', 'afterHook 1') }) - describe('open in IDE', () => { - beforeEach(() => { - loadSpec({ - fileName: 'basic.cy.js', - passCount: 2, - hasPreferredIde: true, - }) + it('creates open in IDE button', () => { + loadSpec({ + fileName: 'basic.cy.js', + passCount: 2, + hasPreferredIde: true, }) - it('creates open in IDE button', () => { - cy.contains('tests 1').click() + cy.contains('tests 1').click() - cy.get('.hook-open-in-ide').should('have.length', 4) + cy.get('.hook-open-in-ide').should('have.length', 4) - cy.intercept('mutation-OpenFileInIDE', { data: { 'openFileInIDE': true } }).as('OpenIDE') + cy.intercept('mutation-OpenFileInIDE', { data: { 'openFileInIDE': true } }).as('OpenIDE') - cy.contains('Open in IDE').invoke('show').click({ force: true }) + cy.contains('Open in IDE').invoke('show').click({ force: true }) - cy.wait('@OpenIDE').then(({ request }) => { - expect(request.body.variables.input.absolute).to.include('hooks/basic.cy.js') - expect(request.body.variables.input.column).to.eq(Cypress.browser.family === 'firefox' ? 6 : 3) - expect(request.body.variables.input.line).to.eq(2) - }) + cy.wait('@OpenIDE').then(({ request }) => { + expect(request.body.variables.input.absolute).to.include('hooks/basic.cy.js') + expect(request.body.variables.input.column).to.eq(Cypress.browser.family === 'firefox' ? 6 : 3) + expect(request.body.variables.input.line).to.eq(2) }) }) - describe('skipped tests', () => { - beforeEach(() => { - loadSpec({ - fileName: 'skip.cy.js', - passCount: 1, - }) + it('does not display commands from skipped tests', () => { + loadSpec({ + fileName: 'skip.cy.js', + passCount: 1, }) - it('does not display commands from skipped tests', () => { - // does not display commands from skipped tests - cy.contains('test 1').click() - cy.contains('test 1').parents('.collapsible').first().should('not.contain', 'testBody 1') - cy.contains('test 1').click() + // does not display commands from skipped tests + cy.contains('test 1').click() + cy.contains('test 1').parents('.collapsible').first().should('not.contain', 'testBody 1') + cy.contains('test 1').click() - // displays before hook when following it.skip - // https://github.com/cypress-io/cypress/issues/8086 - cy.contains('test 2').click() + // displays before hook when following it.skip + // https://github.com/cypress-io/cypress/issues/8086 + cy.contains('test 2').click() - cy.contains('test 2').parents('.collapsible').first().should('contain', 'before all') - }) + cy.contains('test 2').parents('.collapsible').first().should('contain', 'before all') }) - describe('only tests', () => { - beforeEach(() => { - loadSpec({ - fileName: 'only.cy.js', - passCount: 1, - }) + it('only displays tests with .only', () => { + loadSpec({ + fileName: 'only.cy.js', + passCount: 1, + }) + + cy.contains('test wrapper').parents('.collapsible').first().should(($suite) => { + expect($suite).not.to.contain('test 1') + expect($suite).to.contain('nested suite 1') + expect($suite).to.contain('test 2') + expect($suite).not.to.contain('nested suite 2') + expect($suite).not.to.contain('test 3') + expect($suite).to.contain('nested suite 3') + expect($suite).to.contain('test 4') }) - it('only displays tests with .only', () => { - cy.contains('test wrapper').parents('.collapsible').first().should(($suite) => { - expect($suite).not.to.contain('test 1') - expect($suite).to.contain('nested suite 1') - expect($suite).to.contain('test 2') - expect($suite).not.to.contain('nested suite 2') - expect($suite).not.to.contain('test 3') - expect($suite).to.contain('nested suite 3') - expect($suite).to.contain('test 4') - }) - - cy.contains('test 2').click() - - cy.contains('test 2').parents('.collapsible').first().should(($test) => { - expect($test).to.contain('before each') - expect($test).to.contain('test body') - }) + cy.contains('test 2').click() + + cy.contains('test 2').parents('.collapsible').first().should(($test) => { + expect($test).to.contain('before each') + expect($test).to.contain('test body') }) }) // https://github.com/cypress-io/cypress/issues/8189 - describe('rerun', () => { - it('can rerun without timeout error leaking into next run (due to run restart)', () => { - loadSpec({ - fileName: 'rerun.cy.js', - passCount: 1, - }) - - // wait until spec has run twice (due to one reload) - cy.window().its('count').should('eq', 2) + it('can rerun without timeout error leaking into next run (due to run restart)', () => { + loadSpec({ + fileName: 'rerun.cy.js', + passCount: 1, }) + + // wait until spec has run twice (due to one reload) + cy.window().its('count').should('eq', 2) }) }) diff --git a/packages/app/cypress/e2e/runner/retries.ui.cy.ts b/packages/app/cypress/e2e/runner/retries.ui.cy.ts index 54484a3b02c7..5e95ebaef105 100644 --- a/packages/app/cypress/e2e/runner/retries.ui.cy.ts +++ b/packages/app/cypress/e2e/runner/retries.ui.cy.ts @@ -1,29 +1,10 @@ import { loadSpec } from './support/spec-loader' -const getAttemptTag = (sel) => { +// Returns wrapped attempt tag found within runnable containing selector +const getAttemptTag = (sel: string) => { return cy.get(`.test.runnable:contains(${sel}) .attempt-tag`) } -const shouldBeOpen = ($el) => cy.wrap($el).parentsUntil('.collapsible').last().parent().should('have.class', 'is-open') - -const attemptTag = (sel) => `.attempt-tag:contains(Attempt ${sel})` - -const containText = (text) => { - return (($el) => { - expect($el[0]).property('innerText').contain(text) - }) -} - -// const cyReject = (fn) => { -// return () => { -// try { -// fn() -// } catch (e) { -// cy.state('reject')(e) -// } -// } -// } - describe('retries ui', { viewportWidth: 1024, viewportHeight: 900, @@ -73,37 +54,48 @@ describe('retries ui', { cy.percySnapshot() }) - // TODO: fix this one - // it.only('opens attempt on each attempt failure for the screenshot, and closes after test passes', () => { - // let stub - - // loadSpec({ - // fileName: 'opens-attempt-for-screenshot.retries.cy.js', - // passCount: 3, - // failCount: 0, - // setup: () => { - // let attempt = 0 - - // stub = cy.stub().callsFake(cyReject(() => { - // attempt++ - // expect(cy.$$('.attempt-item > .is-open').length).to.equal(attempt) - // })) - - // debugger - // cy.window().then((win) => { - // debugger - // win.Cypress.Screenshot.onAfterScreenshot = stub - // }) - // }, - // }) - - // cy.get('.test.runnable:contains(t2)').then(($el) => { - // expect($el).not.class('is-open') - // expect(stub).callCount(3) - // }) - // }) + // TODO: determine another way to cover this + it.skip('opens attempt on each attempt failure for the screenshot, and closes after test passes', () => { + let stub + const cyReject = (fn) => { + return () => { + try { + fn() + } catch (e) { + // @ts-ignore + cy.state('reject')(e) + } + } + } + + loadSpec({ + fileName: 'opens-attempt-for-screenshot.retries.cy.js', + passCount: 3, + failCount: 0, + setup: () => { + let attempt = 0 + + stub = cy.stub().callsFake(cyReject(() => { + attempt++ + expect(cy.$$('.attempt-item > .is-open').length).to.equal(attempt) + })) + + cy.window().then((win) => { + // @ts-ignore + win.Cypress.Screenshot.onAfterScreenshot = stub + }) + }, + }) + + cy.get('.test.runnable:contains(t2)').then(($el) => { + expect($el).not.class('is-open') + expect(stub).callCount(3) + }) + }) it('includes routes, spies, hooks, and commands in attempt', () => { + const attemptTag = (sel) => `.attempt-tag:contains(Attempt ${sel})` + loadSpec({ fileName: 'includes-all-in-attempt.retries.cy.js', passCount: 1, @@ -228,6 +220,8 @@ describe('retries ui', { describe('afterEach', () => { it('afterEach hooks retry on failure, but only run higher-level afterEach hooks', () => { + const shouldBeOpen = ($el) => cy.wrap($el).parentsUntil('.collapsible').last().parent().should('have.class', 'is-open') + loadSpec({ fileName: 'after-each-failure.retries.cy.js', passCount: 1, @@ -307,11 +301,24 @@ describe('retries ui', { describe('can configure retries', () => { after(() => { + // @ts-ignore window.top.__cySkipValidateConfig = false }) - const haveCorrectError = ($el) => cy.wrap($el).last().parentsUntil('.collapsible').last().parent().find('.runnable-err').should('contain', 'Unspecified AssertionError') + const haveCorrectError = ($el) => { + return ( + cy.wrap($el).last().parentsUntil('.collapsible').last().parent() + .find('.runnable-err').should('contain', 'Unspecified AssertionError') + ) + } + const containText = (text) => { + return (($el) => { + expect($el[0]).property('innerText').contain(text) + }) + } + + // @ts-ignore window.top.__cySkipValidateConfig = true it('via config value', () => { @@ -337,7 +344,8 @@ describe('retries ui', { failCount: 1, }) - cy.get('.runnable-err').should(containText(`it('tries to set mocha retries', { retries: 2 }, () => `)) + cy.get('.runnable-err') + .should(containText(`it('tries to set mocha retries', { retries: 2 }, () => `)) cy.percySnapshot() }) @@ -349,7 +357,8 @@ describe('retries ui', { failCount: 1, }) - cy.get('.runnable-err').should(containText(`describe('suite 1', { retries: 0 }, () => `)) + cy.get('.runnable-err') + .should(containText(`describe('suite 1', { retries: 0 }, () => `)) cy.percySnapshot() }) From cb0e1e525baa1d16d17a4fff86cc132ef38e6e69 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Wed, 9 Feb 2022 12:59:17 -0600 Subject: [PATCH 25/64] Fixing ts errors --- packages/app/index.d.ts | 3 ++- packages/app/src/runner/index.ts | 2 +- packages/frontend-shared/cypress/e2e/support/e2eSupport.ts | 1 + .../projects/runner-e2e-specs/cypress/e2e/hooks/rerun.cy.js | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/app/index.d.ts b/packages/app/index.d.ts index 305cf68013d9..1d3deb7a2537 100644 --- a/packages/app/index.d.ts +++ b/packages/app/index.d.ts @@ -1,5 +1,6 @@ import type { Socket } from '@packages/socket/lib/browser' import type MobX from 'mobx' +import type { EventManager } from './src/runner/event-manager' export {} @@ -19,7 +20,7 @@ export {} declare global { interface Window { ws: Socket - + getEventManager: () => EventManager UnifiedRunner: { /** * decode config, which we receive as a base64 string diff --git a/packages/app/src/runner/index.ts b/packages/app/src/runner/index.ts index 1073c67d0d3c..edd0b94c071b 100644 --- a/packages/app/src/runner/index.ts +++ b/packages/app/src/runner/index.ts @@ -64,7 +64,7 @@ export function getEventManager () { return _eventManager } -window._getEventManager = getEventManager +window.getEventManager = getEventManager let _autIframeModel: AutIframe diff --git a/packages/frontend-shared/cypress/e2e/support/e2eSupport.ts b/packages/frontend-shared/cypress/e2e/support/e2eSupport.ts index 9b404fc1268d..02611465ddae 100644 --- a/packages/frontend-shared/cypress/e2e/support/e2eSupport.ts +++ b/packages/frontend-shared/cypress/e2e/support/e2eSupport.ts @@ -437,6 +437,7 @@ Cypress.Commands.add('findBrowsers', findBrowsers) Cypress.Commands.add('validateExternalLink', { prevSubject: ['optional', 'element'] }, validateExternalLink) installCustomPercyCommand({ + before: () => {}, elementOverrides: { '.runnable-header .duration': ($el) => $el.text('XX:XX'), '.cy-tooltip': true, diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/rerun.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/rerun.cy.js index 028941506068..4074730f3d31 100644 --- a/system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/rerun.cy.js +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/rerun.cy.js @@ -12,7 +12,7 @@ describe('s1', { it('foo', () => { cy.once('test:after:run', () => { if (!top.count) { - top._getEventManager().reporterBus.emit('runner:restart') + top.getEventManager().reporterBus.emit('runner:restart') } top.count++ From bb0b722481197252b8e34e58f24ac0e0719aeb0d Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Wed, 9 Feb 2022 13:11:34 -0600 Subject: [PATCH 26/64] Removing tests for runner header that have existing coverage in app --- .../app/cypress/e2e/runner/runner.ui.cy.ts | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/packages/app/cypress/e2e/runner/runner.ui.cy.ts b/packages/app/cypress/e2e/runner/runner.ui.cy.ts index 5c0888bdff2b..67681defe258 100644 --- a/packages/app/cypress/e2e/runner/runner.ui.cy.ts +++ b/packages/app/cypress/e2e/runner/runner.ui.cy.ts @@ -183,42 +183,6 @@ describe('runner ui', () => { }) }) - // TODO: determine coverage provided by inflight changes - // describe('runner header', () => { - // context('viewport dropdown', () => { - // it('shows on click', () => { - // runIsolatedCypress({}) - // cy.get('.viewport-menu').should('not.be.visible') - // cy.get('.viewport-info button').click() - // cy.get('.viewport-menu').should('be.visible') - - // cy.percySnapshot() - // }) - // }) - - // context('selector playground', () => { - // it('shows on click', () => { - // runIsolatedCypress({}) - - // cy.get('.selector-playground').should('not.be.visible') - // cy.get('.selector-playground-toggle').click() - // cy.get('.selector-playground').should('be.visible') - - // cy.percySnapshot() - // }) - - // it('closes on restart', () => { - // runIsolatedCypress({}) - - // cy.get('.selector-playground-toggle').click() - // cy.get('.selector-playground').should('be.visible') - - // cy.get('.restart').click() - // cy.get('.selector-playground').should('not.be.visible') - // }) - // }) - // }) - describe('reporter interaction', () => { // https://github.com/cypress-io/cypress/issues/8621 it('user can stop test execution', () => { From f8dbc54f48582eb62ba1ad997d648e671f18c251 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Wed, 9 Feb 2022 14:40:03 -0600 Subject: [PATCH 27/64] Working around detached elements in CI --- packages/app/cypress/e2e/runner/sessions.ui.cy.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/app/cypress/e2e/runner/sessions.ui.cy.ts b/packages/app/cypress/e2e/runner/sessions.ui.cy.ts index ed21cf3fe06d..67e6d46e9987 100644 --- a/packages/app/cypress/e2e/runner/sessions.ui.cy.ts +++ b/packages/app/cypress/e2e/runner/sessions.ui.cy.ts @@ -25,7 +25,11 @@ describe('sessions ui', { passCount: 3, }) - cy.get('.test').each(($el) => cy.wrap($el).click()) + const clickEl = ($el) => cy.wrap($el).click() + + cy.get('.test').eq(0).then(clickEl) + cy.get('.test').eq(1).then(clickEl) + cy.get('.test').eq(2).then(clickEl) cy.get('.sessions-container').eq(0).click() .should('contain', '1') From 9c11c4912164fefc679acd7e13e413212aafb5b6 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Wed, 9 Feb 2022 21:45:27 -0600 Subject: [PATCH 28/64] Still working around detached elements in CI --- packages/app/cypress/e2e/runner/sessions.ui.cy.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/app/cypress/e2e/runner/sessions.ui.cy.ts b/packages/app/cypress/e2e/runner/sessions.ui.cy.ts index 67e6d46e9987..6413ea4fb1ce 100644 --- a/packages/app/cypress/e2e/runner/sessions.ui.cy.ts +++ b/packages/app/cypress/e2e/runner/sessions.ui.cy.ts @@ -32,10 +32,10 @@ describe('sessions ui', { cy.get('.test').eq(2).then(clickEl) cy.get('.sessions-container').eq(0).click() - .should('contain', '1') + cy.get('.sessions-container').eq(0).should('contain', '1') cy.get('.sessions-container').eq(1).click() - .should('contain', '1') + cy.get('.sessions-container').eq(1).should('contain', '1') cy.get('.test').eq(0) .should('contain', 'Sessions (1)') From b3dc1c8604892f6113a352a02980e3eaf183f110 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Wed, 9 Feb 2022 22:22:43 -0600 Subject: [PATCH 29/64] Updating sessions.ui test to click more carefully. Adding bit more coverage around runner header and snapshot pinning. --- .../app/cypress/e2e/runner/runner.ui.cy.ts | 35 +++++++++++++++++++ .../app/cypress/e2e/runner/sessions.ui.cy.ts | 8 ++--- .../e2e/runner/simple-cy-assert.runner.cy.js | 9 +++++ 3 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 system-tests/projects/runner-e2e-specs/cypress/e2e/runner/simple-cy-assert.runner.cy.js diff --git a/packages/app/cypress/e2e/runner/runner.ui.cy.ts b/packages/app/cypress/e2e/runner/runner.ui.cy.ts index 67681defe258..c73f9bf8e3b5 100644 --- a/packages/app/cypress/e2e/runner/runner.ui.cy.ts +++ b/packages/app/cypress/e2e/runner/runner.ui.cy.ts @@ -80,6 +80,41 @@ describe('runner ui', () => { }) }) + it('pins cy assertion when clicked', () => { + loadSpec({ + fileName: 'simple-cy-assert.runner.cy.js', + passCount: 1, + failCount: 0, + }) + + cy.contains('li.command-name-assert.command-has-snapshot', 'assert') + .should('not.have.class', 'command-is-pinned') + .click() + .should('have.class', 'command-is-pinned') + + cy.percySnapshot() + }) + + it('renders spec name and runtime in header', () => { + loadSpec({ + fileName: 'simple-cy-assert.runner.cy.js', + passCount: 1, + failCount: 0, + hasPreferredIde: true, + }) + + cy.intercept('mutation-OpenFileInIDE', { data: { 'openFileInIDE': true } }).as('OpenIDE') + + cy.contains('a', 'simple-cy-assert.runner') + .click() + + cy.wait('@OpenIDE').then(({ request }) => { + expect(request.body.variables.input.absolute).to.include('simple-cy-assert.runner.cy.js') + }) + + cy.get('[data-cy="runnable-header"] [data-cy="spec-duration"]').should('exist') + }) + describe('hook failures', () => { describe('test failures w/ hooks', () => { it('test [only]', () => { diff --git a/packages/app/cypress/e2e/runner/sessions.ui.cy.ts b/packages/app/cypress/e2e/runner/sessions.ui.cy.ts index 6413ea4fb1ce..85c85bc7800e 100644 --- a/packages/app/cypress/e2e/runner/sessions.ui.cy.ts +++ b/packages/app/cypress/e2e/runner/sessions.ui.cy.ts @@ -31,11 +31,11 @@ describe('sessions ui', { cy.get('.test').eq(1).then(clickEl) cy.get('.test').eq(2).then(clickEl) - cy.get('.sessions-container').eq(0).click() - cy.get('.sessions-container').eq(0).should('contain', '1') + cy.get('.sessions-container .collapsible-header[role=button]').eq(0).click() + .should('contain', '1') - cy.get('.sessions-container').eq(1).click() - cy.get('.sessions-container').eq(1).should('contain', '1') + cy.get('.sessions-container .collapsible-header[role=button]').eq(1).click() + .should('contain', '1') cy.get('.test').eq(0) .should('contain', 'Sessions (1)') diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/simple-cy-assert.runner.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/simple-cy-assert.runner.cy.js new file mode 100644 index 000000000000..089a159dd7a6 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/runner/simple-cy-assert.runner.cy.js @@ -0,0 +1,9 @@ +describe('cy assertion', { + numTestsKeptInMemory: 1, +}, () => { + it('visits fixture', () => { + cy.visit('cypress/fixtures/index.html') + + cy.get('html').should('exist') + }) +}) From 4df910c97cebed801564b01a61d4fd3f0483cb1a Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Thu, 10 Feb 2022 16:49:36 -0600 Subject: [PATCH 30/64] Update packages/app/cypress/e2e/runner/retries.ui.cy.ts Co-authored-by: Ryan Manuel --- packages/app/cypress/e2e/runner/retries.ui.cy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/app/cypress/e2e/runner/retries.ui.cy.ts b/packages/app/cypress/e2e/runner/retries.ui.cy.ts index 5e95ebaef105..b00b600c17f3 100644 --- a/packages/app/cypress/e2e/runner/retries.ui.cy.ts +++ b/packages/app/cypress/e2e/runner/retries.ui.cy.ts @@ -267,7 +267,7 @@ describe('retries ui', { }) describe('afterAll', () => { - it('only run afterAll hook on last attempt', () => { + it('only runs afterAll hook on last attempt', () => { loadSpec({ fileName: 'after-all-once.retries.cy.js', passCount: 3, From f1431ab7ee1fc64f6fd251a1ba25cd637cc3bd7e Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Thu, 10 Feb 2022 23:02:18 -0600 Subject: [PATCH 31/64] Migrating issue specs. Updating TODOs. Hopefully addressing some flake. --- .../cypress/e2e/runner/issues/issue-18042.cy.js} | 14 ++++++-------- .../cypress/e2e/runner/issues/issue-9162.cy.js | 14 ++++++++++++++ packages/app/cypress/e2e/runner/retries.ui.cy.ts | 3 ++- packages/app/cypress/e2e/runner/runner.ui.cy.ts | 2 +- packages/runner/cypress/e2e/issues/issue-9162.js | 15 --------------- .../cypress/e2e/hooks/rerun.cy.js | 10 ++++++++-- .../cypress/e2e/issues/issue-18042.cy.js | 0 .../cypress/e2e/issues/issue-9162.cy.js | 0 8 files changed, 31 insertions(+), 27 deletions(-) rename packages/{runner/cypress/e2e/issues/issue-18042.js => app/cypress/e2e/runner/issues/issue-18042.cy.js} (53%) create mode 100644 packages/app/cypress/e2e/runner/issues/issue-9162.cy.js delete mode 100644 packages/runner/cypress/e2e/issues/issue-9162.js rename packages/runner/cypress/fixtures/issues/issue-18042.js => system-tests/projects/runner-e2e-specs/cypress/e2e/issues/issue-18042.cy.js (100%) rename packages/runner/cypress/fixtures/issues/issue-9162.js => system-tests/projects/runner-e2e-specs/cypress/e2e/issues/issue-9162.cy.js (100%) diff --git a/packages/runner/cypress/e2e/issues/issue-18042.js b/packages/app/cypress/e2e/runner/issues/issue-18042.cy.js similarity index 53% rename from packages/runner/cypress/e2e/issues/issue-18042.js rename to packages/app/cypress/e2e/runner/issues/issue-18042.cy.js index a668a281a131..4700fce3dbdb 100644 --- a/packages/runner/cypress/e2e/issues/issue-18042.js +++ b/packages/app/cypress/e2e/runner/issues/issue-18042.cy.js @@ -1,15 +1,13 @@ -const helpers = require('../../support/helpers') - -const { createCypress } = helpers -const { runIsolatedCypress } = createCypress() +import { loadSpec } from '../support/spec-loader' // https://github.com/cypress-io/cypress/issues/18042 describe('issue 18042', () => { - beforeEach(function () { - return runIsolatedCypress(`cypress/fixtures/issues/issue-18042.js`) - }) - it('Call count is shown even if cy.stub().log(false)', function () { + loadSpec({ + fileName: 'issue-18042.cy.js', + passCount: 1, + }) + cy.contains('Spies / Stubs (1)').click() cy.get('.call-count').eq(1).should('have.text', '1') }) diff --git a/packages/app/cypress/e2e/runner/issues/issue-9162.cy.js b/packages/app/cypress/e2e/runner/issues/issue-9162.cy.js new file mode 100644 index 000000000000..76b61619975e --- /dev/null +++ b/packages/app/cypress/e2e/runner/issues/issue-9162.cy.js @@ -0,0 +1,14 @@ +import { loadSpec } from '../support/spec-loader' + +// https://github.com/cypress-io/cypress/issues/9162 +describe('issue 9162', () => { + it('tests does not hang even if there is a fail in before().', function () { + loadSpec({ + fileName: 'issue-9162.cy.js', + passCount: 0, + failCount: 1, + }) + + cy.contains('expected true to be false') + }) +}) diff --git a/packages/app/cypress/e2e/runner/retries.ui.cy.ts b/packages/app/cypress/e2e/runner/retries.ui.cy.ts index b00b600c17f3..64194a636120 100644 --- a/packages/app/cypress/e2e/runner/retries.ui.cy.ts +++ b/packages/app/cypress/e2e/runner/retries.ui.cy.ts @@ -54,7 +54,8 @@ describe('retries ui', { cy.percySnapshot() }) - // TODO: determine another way to cover this + // TODO: Determine how best to access the inner Cypress instance prior to test execution + // spy on its actions during the test run. it.skip('opens attempt on each attempt failure for the screenshot, and closes after test passes', () => { let stub const cyReject = (fn) => { diff --git a/packages/app/cypress/e2e/runner/runner.ui.cy.ts b/packages/app/cypress/e2e/runner/runner.ui.cy.ts index c73f9bf8e3b5..5315c0cad04e 100644 --- a/packages/app/cypress/e2e/runner/runner.ui.cy.ts +++ b/packages/app/cypress/e2e/runner/runner.ui.cy.ts @@ -230,7 +230,7 @@ describe('runner ui', () => { cy.get('.runnable-err-message').should('not.contain', 'ran afterEach even though specs were stopped') }) - // TODO: determine intended function in new runner + // TODO: blocked by UNIFY-1077 // it('supports disabling command log reporter with env var NO_COMMAND_LOG', () => { // loadSpec({ // fileName: 'disabled-command-log.runner.cy.js', diff --git a/packages/runner/cypress/e2e/issues/issue-9162.js b/packages/runner/cypress/e2e/issues/issue-9162.js deleted file mode 100644 index 784944182c21..000000000000 --- a/packages/runner/cypress/e2e/issues/issue-9162.js +++ /dev/null @@ -1,15 +0,0 @@ -const helpers = require('../../support/helpers') - -const { createCypress } = helpers -const { runIsolatedCypress } = createCypress() - -// https://github.com/cypress-io/cypress/issues/9162 -describe('issue 9162', () => { - beforeEach(function () { - return runIsolatedCypress(`cypress/fixtures/issues/issue-9162.js`) - }) - - it('tests does not hang even if there is a fail in before().', function () { - cy.contains('expected true to be false') - }) -}) diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/rerun.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/rerun.cy.js index 4074730f3d31..85bebaf85e0b 100644 --- a/system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/rerun.cy.js +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/hooks/rerun.cy.js @@ -6,13 +6,19 @@ describe('s1', { defaultCommandTimeout: 50, }, () => { afterEach(function () { - assert(true, `run ${top.count}`) + if (!top.count) { + assert(false) + } else { + assert(true, `run ${top.count}`) + } }) it('foo', () => { cy.once('test:after:run', () => { if (!top.count) { - top.getEventManager().reporterBus.emit('runner:restart') + requestAnimationFrame(() => { + top.getEventManager().reporterBus.emit('runner:restart') + }) } top.count++ diff --git a/packages/runner/cypress/fixtures/issues/issue-18042.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/issues/issue-18042.cy.js similarity index 100% rename from packages/runner/cypress/fixtures/issues/issue-18042.js rename to system-tests/projects/runner-e2e-specs/cypress/e2e/issues/issue-18042.cy.js diff --git a/packages/runner/cypress/fixtures/issues/issue-9162.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/issues/issue-9162.cy.js similarity index 100% rename from packages/runner/cypress/fixtures/issues/issue-9162.js rename to system-tests/projects/runner-e2e-specs/cypress/e2e/issues/issue-9162.cy.js From cb6303b4607c3e51720f6ddbc55659d5d96fda16 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Fri, 11 Feb 2022 12:32:06 -0600 Subject: [PATCH 32/64] Stopping icon spin animation for percy snapshots --- packages/launchpad/cypress/component/support/index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/launchpad/cypress/component/support/index.ts b/packages/launchpad/cypress/component/support/index.ts index 9bc72c22283a..f867e4a6b789 100644 --- a/packages/launchpad/cypress/component/support/index.ts +++ b/packages/launchpad/cypress/component/support/index.ts @@ -26,4 +26,11 @@ import './commands' import './attachFileWithPath' import installCustomPercyCommand from '@packages/ui-components/cypress/support/customPercyCommand' -installCustomPercyCommand() +installCustomPercyCommand({ + before: () => {}, + elementOverrides: { + 'svg.animate-spin': ($el) => { + $el.attr('style', 'animation: none !important') + }, + }, +}) From f11a98a078f6bf960d7df38eff42379628cf79e9 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Fri, 11 Feb 2022 14:49:11 -0600 Subject: [PATCH 33/64] WIP --- packages/app/cypress/e2e/support/e2eSupport.ts | 11 +++++++++++ .../frontend-shared/cypress/e2e/support/e2eSupport.ts | 8 +------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/app/cypress/e2e/support/e2eSupport.ts b/packages/app/cypress/e2e/support/e2eSupport.ts index 6f190a69ad06..bd24857580e2 100644 --- a/packages/app/cypress/e2e/support/e2eSupport.ts +++ b/packages/app/cypress/e2e/support/e2eSupport.ts @@ -1,2 +1,13 @@ import '@packages/frontend-shared/cypress/e2e/support/e2eSupport' import 'cypress-real-events/support' +import installCustomPercyCommand from '@packages/ui-components/cypress/support/customPercyCommand' + +installCustomPercyCommand({ + before: () => {}, + elementOverrides: { + '.runnable-header .duration': ($el) => $el.text('XX:XX'), + '.cy-tooltip': true, + 'iframe.aut-iframe': true, + '#spec-runner-header': true, + }, +}) diff --git a/packages/frontend-shared/cypress/e2e/support/e2eSupport.ts b/packages/frontend-shared/cypress/e2e/support/e2eSupport.ts index fe68f7d53c65..4c1466f6c6e6 100644 --- a/packages/frontend-shared/cypress/e2e/support/e2eSupport.ts +++ b/packages/frontend-shared/cypress/e2e/support/e2eSupport.ts @@ -523,12 +523,6 @@ Cypress.Commands.add('findBrowsers', findBrowsers) Cypress.Commands.add('tabUntil', tabUntil) Cypress.Commands.add('validateExternalLink', { prevSubject: ['optional', 'element'] }, validateExternalLink) -installCustomPercyCommand({ - before: () => {}, - elementOverrides: { - '.runnable-header .duration': ($el) => $el.text('XX:XX'), - '.cy-tooltip': true, - }, -}) +installCustomPercyCommand() addNetworkCommands() From 7462254abe2a41cc8198352b691cc37953d34148 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Fri, 11 Feb 2022 16:56:34 -0600 Subject: [PATCH 34/64] customPercyCommand.ts --- ...mPercyCommand.js => customPercyCommand.ts} | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) rename packages/ui-components/cypress/support/{customPercyCommand.js => customPercyCommand.ts} (79%) diff --git a/packages/ui-components/cypress/support/customPercyCommand.js b/packages/ui-components/cypress/support/customPercyCommand.ts similarity index 79% rename from packages/ui-components/cypress/support/customPercyCommand.js rename to packages/ui-components/cypress/support/customPercyCommand.ts index 7145be07ab7a..c12d727c3dcf 100644 --- a/packages/ui-components/cypress/support/customPercyCommand.js +++ b/packages/ui-components/cypress/support/customPercyCommand.ts @@ -1,7 +1,18 @@ require('@percy/cypress') const _ = require('lodash') -const installCustomPercyCommand = ({ before, elementOverrides } = {}) => { +declare namespace Cypress { + interface Chainable { + percySnapshot( + name?: string, + options?: SnapshotOptions & { + elementOverrides: any + } + ): Chainable + } +} + +export const installCustomPercyCommand = ({ before, elementOverrides } = {}) => { const customPercySnapshot = (origFn, name, options = {}) => { if (_.isObject(name)) { options = name @@ -9,7 +20,10 @@ const installCustomPercyCommand = ({ before, elementOverrides } = {}) => { } const opts = _.defaults({}, options, { - elementOverrides, + elementOverrides: { + ...elementOverrides, + ...options.elementOverrides, + }, widths: [Cypress.config().viewportWidth], }) @@ -65,5 +79,3 @@ const installCustomPercyCommand = ({ before, elementOverrides } = {}) => { Cypress.Commands.overwrite('percySnapshot', customPercySnapshot) } - -module.exports = installCustomPercyCommand From 232f13f7668932b424b9fc67bdf5465693cf2486 Mon Sep 17 00:00:00 2001 From: Brian Mann Date: Fri, 11 Feb 2022 18:57:02 -0500 Subject: [PATCH 35/64] WIP: working on custom percy snapshots for hiding elements other than the reporter [skip ci] --- .../app/cypress/e2e/runner/sessions.ui.cy.ts | 10 +++- .../app/cypress/e2e/support/e2eSupport.ts | 2 +- .../app/src/navigation/SidebarNavigation.vue | 1 + packages/app/src/runner/ResizablePanels.vue | 1 + .../cypress/e2e/support/e2eProjectDirs.ts | 1 + .../cypress/e2e/support/e2eSupport.ts | 19 ++++--- .../cypress/support/customPercyCommand.ts | 56 ++++++++++++++----- 7 files changed, 65 insertions(+), 25 deletions(-) diff --git a/packages/app/cypress/e2e/runner/sessions.ui.cy.ts b/packages/app/cypress/e2e/runner/sessions.ui.cy.ts index 85c85bc7800e..461641705019 100644 --- a/packages/app/cypress/e2e/runner/sessions.ui.cy.ts +++ b/packages/app/cypress/e2e/runner/sessions.ui.cy.ts @@ -16,7 +16,15 @@ describe('sessions ui', { cy.get('.sessions-container').click() .should('contain', 'blank_session') - cy.percySnapshot() + // cy.viewport(600, 600) + + cy.percySnapshot({ + width: 320, + elementOverrides: { + '[data-cy=aut-panel]': true, + '[data-cy=sidebar]': 'displayNone', + }, + }) }) it('shows message for new, saved, and recreated session', () => { diff --git a/packages/app/cypress/e2e/support/e2eSupport.ts b/packages/app/cypress/e2e/support/e2eSupport.ts index bd24857580e2..e3dc8788cf5b 100644 --- a/packages/app/cypress/e2e/support/e2eSupport.ts +++ b/packages/app/cypress/e2e/support/e2eSupport.ts @@ -1,6 +1,6 @@ import '@packages/frontend-shared/cypress/e2e/support/e2eSupport' import 'cypress-real-events/support' -import installCustomPercyCommand from '@packages/ui-components/cypress/support/customPercyCommand' +import { installCustomPercyCommand } from '@packages/ui-components/cypress/support/customPercyCommand' installCustomPercyCommand({ before: () => {}, diff --git a/packages/app/src/navigation/SidebarNavigation.vue b/packages/app/src/navigation/SidebarNavigation.vue index a6f7501c6984..99cd4f8b8565 100644 --- a/packages/app/src/navigation/SidebarNavigation.vue +++ b/packages/app/src/navigation/SidebarNavigation.vue @@ -1,5 +1,6 @@