diff --git a/packages/app/cypress/e2e/runner/reporter.errors.cy.ts b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts new file mode 100644 index 000000000000..30f070e042f4 --- /dev/null +++ b/packages/app/cypress/e2e/runner/reporter.errors.cy.ts @@ -0,0 +1,90 @@ +import { verify } from './support/verify-helpers' + +describe('errors ui', { + viewportHeight: 768, + viewportWidth: 1024, +}, () => { + describe('assertion failures', () => { + beforeEach(() => { + cy.scaffoldProject('runner-e2e-specs') + cy.openProject('runner-e2e-specs') + + // set preferred editor to bypass IDE selection dialog + cy.withCtx((ctx) => { + ctx.coreData.localSettings.availableEditors = [ + ...ctx.coreData.localSettings.availableEditors, + { + id: 'test-editor', + binary: '/usr/bin/test-editor', + name: 'Test editor', + }, + ] + + ctx.coreData.localSettings.preferences.preferredEditorBinary = 'test-editor' + }) + + cy.startAppServer() + cy.visitApp() + + cy.contains('[data-cy=spec-item]', 'assertions.cy.js').click() + + cy.location().should((location) => { + expect(location.hash).to.contain('assertions.cy.js') + }) + + // Wait for specs to complete + cy.findByLabelText('Stats').get('.failed', { timeout: 10000 }).should('have.text', 'Failed:3') + }) + + verify.it('with expect().', { + file: 'assertions.cy.js', + hasPreferredIde: true, + column: 25, + message: `expected 'actual' to equal 'expected'`, + codeFrameText: 'with expect().', + }) + + verify.it('with assert()', { + file: 'assertions.cy.js', + hasPreferredIde: true, + column: '(5|12)', // (chrome|firefox) + message: `should be true`, + codeFrameText: 'with assert()', + }) + + verify.it('with assert.()', { + file: 'assertions.cy.js', + hasPreferredIde: true, + column: 12, + message: `expected 'actual' to equal 'expected'`, + codeFrameText: 'with assert.()', + }) + }) + + describe('assertion failures - no preferred IDE', () => { + beforeEach(() => { + cy.scaffoldProject('runner-e2e-specs') + cy.openProject('runner-e2e-specs') + + cy.startAppServer() + cy.visitApp() + + cy.contains('[data-cy=spec-item]', 'assertions.cy.js').click() + + cy.location().should((location) => { + expect(location.hash).to.contain('assertions.cy.js') + }) + + // Wait for specs to complete + cy.findByLabelText('Stats').get('.failed', { timeout: 10000 }).should('have.text', 'Failed:3') + }) + + verify.it('with expect().', { + file: 'assertions.cy.js', + hasPreferredIde: false, + column: 25, + message: `expected 'actual' to equal 'expected'`, + codeFrameText: 'with expect().', + }) + }) +}) diff --git a/packages/app/cypress/e2e/runner/support/verify-helpers.ts b/packages/app/cypress/e2e/runner/support/verify-helpers.ts new file mode 100644 index 000000000000..dd0a742f9beb --- /dev/null +++ b/packages/app/cypress/e2e/runner/support/verify-helpers.ts @@ -0,0 +1,205 @@ +import _ from 'lodash' +import defaultMessages from '@packages/frontend-shared/src/locales/en-US.json' + +// Assert that either the the dialog is presented or the mutation is emitted, depending on +// whether the test has a preferred IDE defined. +const verifyIdeOpen = ({ file, action, hasPreferredIde }) => { + if (hasPreferredIde) { + cy.intercept('mutation-OpenFileInIDE', { data: { 'openFileInIDE': true } }).as('OpenIDE') + + action() + + cy.wait('@OpenIDE').then(({ request }) => { + expect(request.body.variables.input.absolute).to.include(file) + }) + } else { + action() + + cy.contains(defaultMessages.globalPage.selectPreferredEditor).should('be.visible') + cy.findByRole('button', { name: defaultMessages.actions.close }).click() + } +} + +export const verifyFailure = (options) => { + const { + specTitle, + hasCodeFrame = true, + verifyOpenInIde = true, + hasPreferredIde, + column, + codeFrameText, + originalMessage, + message = [], + notInMessage = [], + command, + stack, + file, + uncaught = false, + uncaughtMessage, + } = options + let { regex, line } = options + + regex = regex || new RegExp(`${file}:${line || '\\d+'}:${column}`) + + cy.contains('.runnable-title', specTitle).closest('.runnable').as('Root') + + cy.get('@Root').within(() => { + cy.contains('View stack trace').click() + + const messageLines = [].concat(message) + + if (messageLines.length) { + cy.log('message contains expected lines and stack does not include message') + + _.each(messageLines, (msg) => { + cy.get('.runnable-err-message') + .should('include.text', msg) + + cy.get('.runnable-err-stack-trace') + .should('not.include.text', msg) + }) + } + + if (originalMessage) { + cy.get('.runnable-err-message') + .should('include.text', originalMessage) + } + + const notInMessageLines = [].concat(notInMessage) + + if (notInMessageLines.length) { + cy.log('message does not contain the specified lines') + + _.each(notInMessageLines, (msg) => { + cy.get('.runnable-err-message') + .should('not.include.text', msg) + }) + } + + cy.log('stack trace matches the specified pattern') + cy.get('.runnable-err-stack-trace') + .invoke('text') + .should('match', regex) + + if (stack) { + const stackLines = [].concat(stack) + + if (stackLines.length) { + cy.log('stack contains the expected lines') + } + + _.each(stackLines, (stackLine) => { + cy.get('.runnable-err-stack-trace') + .should('include.text', stackLine) + }) + } + + cy.get('.runnable-err-stack-trace') + .invoke('text') + .should('not.include', '__stackReplacementMarker') + .should((stackTrace) => { + // if this stack trace has the 'From Your Spec Code' addendum, + // it should only appear once + const match = stackTrace.match(/From Your Spec Code/g) + + if (match && match.length) { + expect(match.length, `'From Your Spec Code' should only be in the stack once, but found ${match.length} instances`).to.equal(1) + } + }) + }) + + if (verifyOpenInIde) { + verifyIdeOpen({ + file, + hasPreferredIde, + action: () => { + cy.get('@Root').contains('.runnable-err-stack-trace .runnable-err-file-path a', file) + .click('left') + }, + }) + } + + cy.get('@Root').within(() => { + if (command) { + cy.log('the error is attributed to the correct command') + cy + .get('.command-state-failed') + .first() + .find('.command-method') + .invoke('text') + .should('equal', command) + } + + if (uncaught) { + cy.log('uncaught error has an associated log for the original error') + cy.get('.command-name-uncaught-exception') + .should('have.length', 1) + .should('have.class', 'command-state-failed') + .find('.command-message-text') + .should('include.text', uncaughtMessage || originalMessage) + } else { + cy.log('"caught" error does not have an uncaught error log') + cy.get('.command-name-uncaught-exception').should('not.exist') + } + + if (!hasCodeFrame) return + + cy.log('code frame matches specified pattern') + cy + .get('.test-err-code-frame .runnable-err-file-path') + .invoke('text') + .should('match', regex) + + cy.get('.test-err-code-frame pre span').should('include.text', codeFrameText) + }) + + if (verifyOpenInIde) { + verifyIdeOpen({ + file, + hasPreferredIde, + action: () => { + cy.get('@Root').contains('.test-err-code-frame .runnable-err-file-path a', file) + .click() + }, + }) + } +} + +const createVerifyTest = (modifier?: string) => { + return (title: string, opts: any, props?: any) => { + if (!props) { + props = opts + opts = null + } + + props.specTitle ||= title + + const verifyFn = props.verifyFn || verifyFailure.bind(null, props) + + return (modifier ? it[modifier] : it)(title, verifyFn) + } +} + +export const verify = { + it: createVerifyTest(), +} + +verify.it['only'] = createVerifyTest('only') +verify.it['skip'] = createVerifyTest('skip') + +export const verifyInternalFailure = (props) => { + const { method, stackMethod } = props + + cy.get('.runnable-err-message') + .should('include.text', `thrown in ${method.replace(/\./g, '-')}`) + + cy.get('.runnable-err-stack-expander > .collapsible-header').click() + + cy.get('.runnable-err-stack-trace') + .should('include.text', stackMethod || method) + + // this is an internal cypress error and we can only show code frames + // from specs, so it should not show the code frame + cy.get('.test-err-code-frame') + .should('not.exist') +} diff --git a/packages/frontend-shared/cypress/e2e/support/e2eProjectDirs.ts b/packages/frontend-shared/cypress/e2e/support/e2eProjectDirs.ts index f19501821537..f7b7a0338fd0 100644 --- a/packages/frontend-shared/cypress/e2e/support/e2eProjectDirs.ts +++ b/packages/frontend-shared/cypress/e2e/support/e2eProjectDirs.ts @@ -85,6 +85,7 @@ export const e2eProjectDirs = [ 'remote-debugging-disconnect', 'remote-debugging-port-removed', 'retries-2', + 'runner-e2e-specs', 'same-fixtures-integration-folders', 'screen-size', 'selectFile', diff --git a/packages/reporter/cypress/support/utils.ts b/packages/reporter/cypress/support/utils.ts index 20a12d51fc47..e0e102c5f7c2 100644 --- a/packages/reporter/cypress/support/utils.ts +++ b/packages/reporter/cypress/support/utils.ts @@ -1,5 +1,4 @@ import { EventEmitter } from 'events' -import { Editor } from '@packages/ui-components' import CommandModel from './../../src/commands/command-model' const { _ } = Cypress @@ -18,131 +17,16 @@ interface HandlesFileOpeningProps { } export const itHandlesFileOpening = ({ getRunner, selector, file, stackTrace = false }: HandlesFileOpeningProps) => { - beforeEach(() => { - cy.stub(getRunner(), 'emit').callThrough() - }) - - describe('when user has already set opener and opens file', () => { - let editor: Partial - - beforeEach(() => { - editor = {} - - // @ts-ignore - getRunner().emit.withArgs('get:user:editor').yields({ - preferredOpener: editor, - }) + describe('it handles file opening', () => { + it('emits unified file open event', () => { + cy.stub(getRunner(), 'emit').callThrough() if (stackTrace) { cy.contains('View stack trace').click() } - }) - it('opens in preferred opener', () => { cy.get(selector).first().click().then(() => { - expect(getRunner().emit).to.be.calledWith('open:file', { - where: editor, - ...file, - }) - }) - }) - }) - - describe('when user has not already set opener and opens file', () => { - const availableEditors = [ - { id: 'computer', name: 'On Computer', isOther: false, binary: 'computer' }, - { id: 'atom', name: 'Atom', isOther: false, binary: 'atom' }, - { id: 'vim', name: 'Vim', isOther: false, binary: 'vim' }, - { id: 'sublime', name: 'Sublime Text', isOther: false, binary: 'sublime' }, - { id: 'vscode', name: 'Visual Studio Code', isOther: false, binary: 'vscode' }, - { id: 'other', name: 'Other', isOther: true, binary: '' }, - ] - - beforeEach(() => { - // @ts-ignore - getRunner().emit.withArgs('get:user:editor').yields({ availableEditors }) - // usual viewport of only reporter is a bit cramped for the modal - cy.viewport(600, 600) - - if (stackTrace) { - cy.contains('View stack trace').click() - } - - cy.get(selector).first().click() - }) - - it('opens modal with available editors', () => { - _.each(availableEditors, ({ name }) => { - cy.contains(name) - }) - - cy.contains('Other') - cy.contains('Set preference and open file') - }) - - // NOTE: this fails because mobx doesn't make the editors observable, so - // the changes to the path don't bubble up correctly. this only happens - // in the Cypress test and not when running the actual app - it.skip('updates "Other" path when typed into', () => { - cy.contains('Other').find('input[type="text"]').type('/absolute/path/to/foo.js') - .should('have.value', '/absolute/path/to/foo.js') - }) - - describe('when editor is not selected', () => { - it('disables submit button', () => { - cy.contains('Set preference and open file') - .should('have.class', 'is-disabled') - .click() - - cy.wrap(getRunner().emit).should('not.to.be.calledWith', 'set:user:editor') - cy.wrap(getRunner().emit).should('not.to.be.calledWith', 'open:file') - }) - - it('shows validation message when hovering over submit button', () => { - cy.get('.editor-picker-modal .submit').trigger('mouseover') - cy.get('.cy-tooltip').last().should('have.text', 'Please select a preference') - }) - }) - - describe('when Other is selected but path is not entered', () => { - beforeEach(() => { - cy.contains('Other').click() - }) - - it('disables submit button', () => { - cy.contains('Set preference and open file') - .should('have.class', 'is-disabled') - .click() - - cy.wrap(getRunner().emit).should('not.to.be.calledWith', 'set:user:editor') - cy.wrap(getRunner().emit).should('not.to.be.calledWith', 'open:file') - }) - - it('shows validation message when hovering over submit button', () => { - cy.get('.editor-picker-modal .submit').trigger('mouseover') - cy.get('.cy-tooltip').last().should('have.text', 'Please enter the path for the "Other" editor') - }) - }) - - describe('when editor is set', () => { - beforeEach(() => { - cy.contains('Visual Studio Code').click() - cy.contains('Set preference and open file').click() - }) - - it('closes modal', function () { - cy.contains('Set preference and open file').should('not.exist') - }) - - it('emits set:user:editor', () => { - expect(getRunner().emit).to.be.calledWith('set:user:editor', availableEditors[4]) - }) - - it('opens file in selected editor', () => { - expect(getRunner().emit).to.be.calledWith('open:file', { - where: availableEditors[4], - ...file, - }) + expect(getRunner().emit).to.be.calledWith('open:file:unified') }) }) }) diff --git a/packages/reporter/src/hooks/hooks.tsx b/packages/reporter/src/hooks/hooks.tsx index df31e1aa57f4..f7e5031f33b7 100644 --- a/packages/reporter/src/hooks/hooks.tsx +++ b/packages/reporter/src/hooks/hooks.tsx @@ -12,7 +12,6 @@ import HookModel, { HookName } from './hook-model' import ArrowRightIcon from '-!react-svg-loader!@packages/frontend-shared/src/assets/icons/arrow-right_x16.svg' import OpenIcon from '-!react-svg-loader!@packages/frontend-shared/src/assets/icons/technology-code-editor_x16.svg' import OpenFileInIDE from '../lib/open-file-in-ide' -import FileOpener from '../lib/file-opener' export interface HookHeaderProps { model: HookModel @@ -30,18 +29,10 @@ export interface HookOpenInIDEProps { } const HookOpenInIDE = ({ invocationDetails }: HookOpenInIDEProps) => { - if ('__vite__' in window) { - return ( - - Open in IDE - - ) - } - return ( - + Open in IDE - + ) } diff --git a/packages/reporter/src/lib/file-name-opener.tsx b/packages/reporter/src/lib/file-name-opener.tsx index e0e8dad5eb8f..17e60080cdc2 100644 --- a/packages/reporter/src/lib/file-name-opener.tsx +++ b/packages/reporter/src/lib/file-name-opener.tsx @@ -1,12 +1,11 @@ import { observer } from 'mobx-react' import React from 'react' +import { FileDetails } from '@packages/types' // @ts-ignore import Tooltip from '@cypress/react-tooltip' -import { FileDetails } from '@packages/types' - -import FileOpener from './file-opener' import TextIcon from '-!react-svg-loader!@packages/frontend-shared/src/assets/icons/document-text_x16.svg' +import OpenFileInIDE from './open-file-in-ide' interface Props { fileDetails: FileDetails @@ -14,18 +13,24 @@ interface Props { hasIcon?: boolean } +/** + * Renders a link-style element that presents a tooltip on hover + * and opens the file in an external editor when selected. + */ const FileNameOpener = observer((props: Props) => { const { displayFile, originalFile, line, column } = props.fileDetails return ( - - {props.hasIcon && ( - - )} - {displayFile || originalFile}{!!line && `:${line}`}{!!column && `:${column}`} - + + e.preventDefault()}> + {props.hasIcon && ( + + )} + {displayFile || originalFile}{!!line && `:${line}`}{!!column && `:${column}`} + + ) diff --git a/packages/reporter/src/lib/file-opener.tsx b/packages/reporter/src/lib/file-opener.tsx deleted file mode 100644 index 4f1363afe376..000000000000 --- a/packages/reporter/src/lib/file-opener.tsx +++ /dev/null @@ -1,43 +0,0 @@ -import { observer } from 'mobx-react' -import React, { ReactNode } from 'react' -import type { FileDetails } from '@packages/types' -import { GetUserEditorResult, Editor, FileOpener as Opener } from '@packages/ui-components' - -import events from './events' - -interface Props { - fileDetails: FileDetails - children: ReactNode - className?: string -} - -const openFile = (where: Editor, { absoluteFile: file, line, column }: FileDetails) => { - events.emit('open:file', { - where, - file, - line, - column, - }) -} - -const getUserEditor = (callback: (result: GetUserEditorResult) => any) => { - events.emit('get:user:editor', callback) -} - -const setUserEditor = (editor: Editor) => { - events.emit('set:user:editor', editor) -} - -const FileOpener = observer(({ fileDetails, children, className }: Props) => ( - - {children} - -)) - -export default FileOpener diff --git a/packages/reporter/src/lib/open-file-in-ide.tsx b/packages/reporter/src/lib/open-file-in-ide.tsx index c916bdb1fce1..cccd0124f4eb 100644 --- a/packages/reporter/src/lib/open-file-in-ide.tsx +++ b/packages/reporter/src/lib/open-file-in-ide.tsx @@ -1,23 +1,20 @@ import { observer } from 'mobx-react' import React from 'react' -// @ts-ignore -import Tooltip from '@cypress/react-tooltip' import type { FileDetails } from '@packages/types' import events from './events' interface Props { fileDetails: FileDetails className?: string - hasIcon?: boolean } +// Catches click events that bubble from children and emits open file events to +// be handled by the app. const OpenFileInIDE: React.FC = observer((props) => { return ( - - events.emit('open:file:unified', props.fileDetails)}> - {props.children} - - + events.emit('open:file:unified', props.fileDetails)}> + {props.children} + ) }) diff --git a/packages/reporter/src/runnables/runnable-header.tsx b/packages/reporter/src/runnables/runnable-header.tsx index ec8d768f78d7..355c5b150dda 100644 --- a/packages/reporter/src/runnables/runnable-header.tsx +++ b/packages/reporter/src/runnables/runnable-header.tsx @@ -3,11 +3,9 @@ import React, { Component, ReactElement } from 'react' import { StatsStore } from '../header/stats-store' import { formatDuration, getFilenameParts } from '../lib/util' -import OpenFileInIDE from '../lib/open-file-in-ide' import FileNameOpener from '../lib/file-name-opener' -import TextIcon from '-!react-svg-loader!@packages/frontend-shared/src/assets/icons/document-text_x16.svg' -const renderRunnableHeader = (children: ReactElement) =>
{children}
+const renderRunnableHeader = (children: ReactElement) =>
{children}
interface RunnableHeaderProps { spec: Cypress.Cypress['spec'] @@ -52,22 +50,11 @@ class RunnableHeader extends Component { relativeFile: relativeSpecPath, } - const openInIde = '__vite__' in window - ? ( - - e.preventDefault()}> - - {fileDetails.displayFile || fileDetails.originalFile} - - - ) - : - return renderRunnableHeader( <> - {openInIde} + {Boolean(statsStore.duration) && ( - {formatDuration(statsStore.duration)} + {formatDuration(statsStore.duration)} )} , ) diff --git a/packages/reporter/src/runnables/runnables.tsx b/packages/reporter/src/runnables/runnables.tsx index 8fff6084d542..a8885f88545f 100644 --- a/packages/reporter/src/runnables/runnables.tsx +++ b/packages/reporter/src/runnables/runnables.tsx @@ -11,7 +11,7 @@ import { RunnablesStore, RunnableArray } from './runnables-store' import statsStore, { StatsStore } from '../header/stats-store' import { Scroller } from '../lib/scroller' import { AppState } from '../lib/app-state' -import FileOpener from '../lib/file-opener' +import OpenFileInIDE from '../lib/open-file-in-ide' import OpenIcon from '-!react-svg-loader!@packages/frontend-shared/src/assets/icons/technology-code-editor_x16.svg' import StudioIcon from '-!react-svg-loader!@packages/frontend-shared/src/assets/icons/object-magic-wand-dark-mode_x16.svg' @@ -49,15 +49,19 @@ const RunnablesEmptyState = ({ spec, eventManager = events }: RunnablesEmptyStat

Cypress could not detect tests in this file.

{ !isAllSpecs && ( <> - -

Open file in IDE

-
+ { + event.preventDefault() + }}> +

Open file in IDE

+
+

Write a test using your preferred text editor.

Create test with Cypress Studio

Use an interactive tool to author a test right here.

diff --git a/system-tests/projects/e2e/cypress/support/util.js b/system-tests/projects/e2e/cypress/support/util.js index e858ae9da5e5..eb1e6a0d38cd 100644 --- a/system-tests/projects/e2e/cypress/support/util.js +++ b/system-tests/projects/e2e/cypress/support/util.js @@ -28,20 +28,6 @@ export const verify = (ctx, options) => { const fileRegex = new RegExp(`${Cypress.spec.relative}:${line}:${column}`) it(`✓ VERIFY`, function () { - const runnerWs = window.top.ws - - cy.stub(window.top.ws, 'emit').callThrough().withArgs('get:user:editor') - .yields({ - preferredOpener: { - id: 'foo-editor', - name: 'Foo', - binary: 'foo-editor', - isOther: false, - }, - }) - - window.top.ws.emit.callThrough().withArgs('open:file') - cy.wrap(Cypress.$(window.top.document.body)) .find('.reporter') .contains(`FAIL - ${getTitle(ctx)}`) @@ -70,12 +56,6 @@ export const verify = (ctx, options) => { .should('not.include.text', '__stackReplacementMarker') cy.contains('.runnable-err-stack-trace .runnable-err-file-path', openInIdePath.relative) - .click() - .should(() => { - expect(runnerWs.emit).to.be.calledWithMatch('open:file', { - file: openInIdePath.absolute, - }) - }) cy .get('.test-err-code-frame .runnable-err-file-path') @@ -85,14 +65,7 @@ export const verify = (ctx, options) => { // code frames will show `fail(this,()=>` as the 1st line cy.get('.test-err-code-frame pre span').should('include.text', 'fail(this,()=>') - cy.contains('.test-err-code-frame .runnable-err-file-path span', openInIdePath.relative) - .click() - .should(() => { - expect(runnerWs.emit.withArgs('open:file')).to.be.calledTwice - expect(runnerWs.emit).to.be.calledWithMatch('open:file', { - file: openInIdePath.absolute, - }) - }) + cy.contains('.test-err-code-frame .runnable-err-file-path', openInIdePath.relative) }) }) } diff --git a/system-tests/projects/runner-e2e-specs/cypress.config.js b/system-tests/projects/runner-e2e-specs/cypress.config.js new file mode 100644 index 000000000000..d12d96d7e3e0 --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress.config.js @@ -0,0 +1,5 @@ +module.exports = { + e2e: { + supportFile: false, + }, +} diff --git a/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/assertions.cy.js b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/assertions.cy.js new file mode 100644 index 000000000000..5c1549fae31a --- /dev/null +++ b/system-tests/projects/runner-e2e-specs/cypress/e2e/errors/assertions.cy.js @@ -0,0 +1,13 @@ +describe('assertion failures', () => { + it('with expect().', () => { + expect('actual').to.equal('expected') + }) + + it('with assert()', () => { + assert(false, 'should be true') + }) + + it('with assert.()', () => { + assert.equal('actual', 'expected') + }) +})