diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index 4b6c466cd41e7..f2eba2ad10543 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -118,7 +118,7 @@ export class FullConfigInternal { updateSnapshots: takeFirst(configCLIOverrides.updateSnapshots, userConfig.updateSnapshots, 'missing'), updateSourceMethod: takeFirst(configCLIOverrides.updateSourceMethod, userConfig.updateSourceMethod, 'patch'), version: require('../../package.json').version, - workers: resolveWorkers(takeFirst(configCLIOverrides.debug ? 1 : undefined, configCLIOverrides.workers, userConfig.workers, '50%')), + workers: resolveWorkers(takeFirst(configCLIOverrides.pause ? 1 : undefined, configCLIOverrides.workers, userConfig.workers, '50%')), webServer: null, }; for (const key in userConfig) { diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index 8c20d80e97cfa..c6aa1b0a71f26 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -31,6 +31,7 @@ export type ConfigCLIOverrides = { maxFailures?: number; outputDir?: string; preserveOutputDir?: boolean; + pause?: boolean; quiet?: boolean; repeatEach?: number; retries?: number; @@ -91,6 +92,7 @@ export type TestInfoErrorImpl = TestInfoError; export type TestPausedPayload = { testId: string; errors: TestInfoErrorImpl[]; + status: TestStatus; }; export type ResumePayload = {}; diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index c48a605a8f57f..0df97a777ca29 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -308,6 +308,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid updateSourceMethod: options.updateSourceMethod, runAgents: options.runAgents, workers: options.workers, + pause: options.pause ? true : undefined, }; if (options.browser) { @@ -323,10 +324,11 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid }); } - if (options.headed || options.debug) + if (options.headed || options.debug || overrides.pause) overrides.use = { headless: false }; if (!options.ui && options.debug) { overrides.debug = true; + overrides.pause = true; process.env.PWDEBUG = '1'; } if (!options.ui && options.trace) { @@ -401,7 +403,7 @@ const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries const testOptions: [string, { description: string, choices?: string[], preset?: string }][] = [ /* deprecated */ ['--browser ', { description: `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")` }], ['-c, --config ', { description: `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"` }], - ['--debug', { description: `Run tests with Playwright Inspector. Shortcut for "PWDEBUG=1" environment variable and "--timeout=0 --max-failures=1 --headed --workers=1" options` }], + ['--debug', { description: `Run tests with Playwright Inspector. Shortcut for "PWDEBUG=1" environment variable and "--timeout=0 --max-failures=1 --headed --workers=1 --pause" options` }], ['--fail-on-flaky-tests', { description: `Fail if any test is flagged as flaky (default: false)` }], ['--forbid-only', { description: `Fail if test.only is called (default: false)` }], ['--fully-parallel', { description: `Run all tests in parallel (default: false)` }], @@ -417,6 +419,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?: ['--output ', { description: `Folder for output artifacts (default: "test-results")` }], ['--only-changed [ref]', { description: `Only run test files that have been changed between 'HEAD' and 'ref'. Defaults to running all uncommitted changes. Only supports Git.` }], ['--pass-with-no-tests', { description: `Makes test run succeed even if no tests were found` }], + ['--pause', { description: `Run tests in headed mode and pause at the end of test execution` }], ['--project ', { description: `Only run tests from the specified list of projects, supports '*' wildcard (default: run all projects)` }], ['--quiet', { description: `Suppress stdio` }], ['--repeat-each ', { description: `Run each test N times (default: 1)` }], diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts index e3fde36e373cf..f94ffebf29bf2 100644 --- a/packages/playwright/src/reporters/base.ts +++ b/packages/playwright/src/reporters/base.ts @@ -359,20 +359,42 @@ export class TerminalReporter implements ReporterV2 { return formatError(this.screen, error); } + formatSingleResult(test: TestCase, result: TestResult, index?: number): string { + return formatSingleResult(this.screen, this.config, test, result, index); + } + writeLine(line?: string) { this.screen.stdout?.write(line ? line + '\n' : '\n'); } } +function formatSingleResult(screen: Screen, config: FullConfig, test: TestCase, result: TestResult, index?: number): string { + const lines: string[] = []; + const header = formatTestHeader(screen, config, test, { indent: ' ', index }); + lines.push(test.outcome() === 'unexpected' ? screen.colors.red(header) : screen.colors.yellow(header)); + if (test.outcome() === 'unexpected') { + const errorDetails = formatResultFailure(screen, test, result, ' '); + if (errorDetails.length > 0) + lines.push(''); + for (const error of errorDetails) + lines.push(error.message, ''); + } + return lines.join('\n'); +} + export function formatFailure(screen: Screen, config: FullConfig, test: TestCase, index?: number, options?: TerminalReporterOptions): string { const lines: string[] = []; - const header = formatTestHeader(screen, config, test, { indent: ' ', index, mode: 'error', includeTestId: options?.includeTestId }); - lines.push(screen.colors.red(header)); + let printedHeader = false; for (const result of test.results) { const resultLines: string[] = []; const errors = formatResultFailure(screen, test, result, ' '); if (!errors.length) continue; + if (!printedHeader) { + const header = formatTestHeader(screen, config, test, { indent: ' ', index, mode: 'error', includeTestId: options?.includeTestId }); + lines.push(screen.colors.red(header)); + printedHeader = true; + } if (result.retry) { resultLines.push(''); resultLines.push(screen.colors.gray(separator(screen, ` Retry #${result.retry}`))); @@ -455,6 +477,12 @@ function quotePathIfNeeded(path: string): string { return path; } +const kReportedSymbol = Symbol('reported'); + +export function markErrorsAsReported(result: TestResult) { + (result as any)[kReportedSymbol] = result.errors.length; +} + export function formatResultFailure(screen: Screen, test: TestCase, result: TestResult, initialIndent: string): ErrorDetails[] { const errorDetails: ErrorDetails[] = []; @@ -469,7 +497,8 @@ export function formatResultFailure(screen: Screen, test: TestCase, result: Test }); } - for (const error of result.errors) { + const reportedIndex = (result as any)[kReportedSymbol] || 0; + for (const error of result.errors.slice(reportedIndex)) { const formattedError = formatError(screen, error); errorDetails.push({ message: indent(formattedError.message, initialIndent), diff --git a/packages/playwright/src/reporters/line.ts b/packages/playwright/src/reporters/line.ts index 3c34f22308d3e..4bcda258f6154 100644 --- a/packages/playwright/src/reporters/line.ts +++ b/packages/playwright/src/reporters/line.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TerminalReporter } from './base'; +import { markErrorsAsReported, TerminalReporter } from './base'; import type { FullResult, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter'; @@ -78,6 +78,23 @@ class LineReporter extends TerminalReporter { this._updateLine(test, result, step.parent); } + async onTestPaused(test: TestCase, result: TestResult) { + // without TTY, user cannot interrupt the pause. let's skip it. + if (!process.stdin.isTTY && !process.env.PW_TEST_DEBUG_REPORTERS) + return; + + if (!process.env.PW_TEST_DEBUG_REPORTERS) + this.screen.stdout.write(`\u001B[1A\u001B[2K`); + + this.writeLine(this.formatSingleResult(test, result, test.outcome() === 'unexpected' ? ++this._failures : undefined)); + markErrorsAsReported(result); + this.writeLine(this.screen.colors.yellow(` Paused ${test.outcome() === 'unexpected' ? 'on error' : 'at test end'}. Press Ctrl+C to end.`) + '\n\n'); + + this._updateLine(test, result, undefined); + + await new Promise(() => {}); + } + override onTestEnd(test: TestCase, result: TestResult) { super.onTestEnd(test, result); if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected' || result.status === 'interrupted')) { diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index dd8defedda16a..db5873fec3913 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -615,6 +615,7 @@ class JobDispatcher { } }; + result.status = params.status; result.errors = params.errors; result.error = result.errors[0]; diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts index c75f75a815172..564347ac780fe 100644 --- a/packages/playwright/src/runner/testRunner.ts +++ b/packages/playwright/src/runner/testRunner.ts @@ -482,7 +482,7 @@ export async function runAllTestsWithConfig(config: FullConfigInternal): Promise ...createRunTestsTasks(config), ]; - const testRun = new TestRun(config, reporter, { pauseAtEnd: config.configCLIOverrides.debug, pauseOnError: config.configCLIOverrides.debug }); + const testRun = new TestRun(config, reporter, { pauseAtEnd: config.configCLIOverrides.pause, pauseOnError: config.configCLIOverrides.pause }); const status = await runTasks(testRun, tasks, config.config.globalTimeout); // Calling process.exit() might truncate large stdout/stderr output. diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index 4d99313c05c1a..db8a4d6e2ec5e 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -474,7 +474,7 @@ export class TestInfoImpl implements TestInfo { const shouldPause = (this._workerParams.pauseAtEnd && !this._isFailure()) || (this._workerParams.pauseOnError && this._isFailure()); if (shouldPause) { await Promise.race([ - this._callbacks.onTestPaused({ testId: this.testId, errors: this._isFailure() ? this.errors : [] }), + this._callbacks.onTestPaused({ testId: this.testId, errors: this._isFailure() ? this.errors : [], status: this.status }), this._interruptedPromise, ]); } diff --git a/tests/playwright-test/pause-at-end.spec.ts b/tests/playwright-test/pause-at-end.spec.ts index 6e47a38fa3f0c..60f27758640ae 100644 --- a/tests/playwright-test/pause-at-end.spec.ts +++ b/tests/playwright-test/pause-at-end.spec.ts @@ -79,7 +79,7 @@ test('--debug should pause at end', async ({ runInlineTest }) => { console.log('%%teardown'); }); ` - }, { debug: true }); + }, { pause: true }); expect(result.outputLines).toEqual([ 'onTestPaused at end', 'teardown', @@ -110,7 +110,7 @@ test('--debug should pause at end with setup project', async ({ runInlineTest }) console.log('main test started'); }); ` - }, { debug: true }); + }, { pause: true }); expect(result.outputLines).toContain('onTestPaused at end'); }); @@ -128,7 +128,7 @@ test('--debug should pause on error', async ({ runInlineTest, mergeReports }) => console.log('%%after error'); }); ` - }, { debug: true }); + }, { pause: true }); expect(result.outputLines).toEqual([ 'onTestPaused on error at :4:24', 'result.errors[0] at :4:24', @@ -159,7 +159,7 @@ test('SIGINT after pause at end should still run teardown', async ({ runInlineTe console.log('%%teardown'); }); ` - }, { debug: true }, { SIGINT_AFTER_PAUSE: '1' }); + }, { pause: true }, { SIGINT_AFTER_PAUSE: '1' }); expect(result.outputLines).toEqual([ 'onTestPaused at end', 'SIGINT', @@ -185,7 +185,7 @@ test('SIGINT after pause on error should still run teardown', async ({ runInline console.log('%%teardown'); }); ` - }, { debug: true }, { SIGINT_AFTER_PAUSE: '1' }); + }, { pause: true }, { SIGINT_AFTER_PAUSE: '1' }); expect(result.outputLines).toEqual([ 'onTestPaused on error at :4:19', 'result.errors[0] at :4:19', diff --git a/tests/playwright-test/reporter-line.spec.ts b/tests/playwright-test/reporter-line.spec.ts index cd6fc6a65c26a..43c42c7081963 100644 --- a/tests/playwright-test/reporter-line.spec.ts +++ b/tests/playwright-test/reporter-line.spec.ts @@ -15,7 +15,7 @@ */ import path from 'path'; -import { test, expect } from './playwright-test-fixtures'; +import { test, expect, stripAnsi } from './playwright-test-fixtures'; for (const useIntermediateMergeReport of [false, true] as const) { test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => { @@ -225,3 +225,151 @@ for (const useIntermediateMergeReport of [false, true] as const) { }); }); } + +test.describe('onTestPaused', () => { + test.skip(process.platform === 'win32', 'No SIGINT on windows'); + + test('pause at end', async ({ interactWithTestRunner }) => { + const runner = await interactWithTestRunner({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('foo', async ({}) => { + }); + + test.afterEach(() => { + console.log('Running teardown'); + }); + `, + }, { pause: true, reporter: 'line' }, { PW_TEST_DEBUG_REPORTERS: '1' }); + + await runner.waitForOutput('Paused at test end. Press Ctrl+C to end.'); + const { exitCode } = await runner.kill('SIGINT'); + expect(exitCode).toBe(130); + + expect(stripAnsi(runner.output)).toEqual(` +Running 1 test using 1 worker + +[1/1] a.test.ts:3:13 › foo + a.test.ts:3:13 › foo ───────────────────────────────────────────────────────────────────────────── + Paused at test end. Press Ctrl+C to end. + + +[1/1] a.test.ts:3:13 › foo +a.test.ts:3:13 › foo +Running teardown + + 1) a.test.ts:3:13 › foo ────────────────────────────────────────────────────────────────────────── + + Test was interrupted. + + + 1 interrupted + a.test.ts:3:13 › foo ─────────────────────────────────────────────────────────────────────────── +`); + }); + + test('pause at end - error in teardown', async ({ interactWithTestRunner }) => { + const runner = await interactWithTestRunner({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('foo', async ({}) => { + }); + + test.afterEach(() => { + throw new Error('teardown error'); + }); + `, + }, { pause: true, reporter: 'line' }, { PW_TEST_DEBUG_REPORTERS: '1' }); + + await runner.waitForOutput('Paused at test end. Press Ctrl+C to end.'); + const { exitCode } = await runner.kill('SIGINT'); + expect(exitCode).toBe(130); + + expect(stripAnsi(runner.output)).toEqual(` +Running 1 test using 1 worker + +[1/1] a.test.ts:3:13 › foo + a.test.ts:3:13 › foo ───────────────────────────────────────────────────────────────────────────── + Paused at test end. Press Ctrl+C to end. + + +[1/1] a.test.ts:3:13 › foo + 1) a.test.ts:3:13 › foo ────────────────────────────────────────────────────────────────────────── + + Test was interrupted. + + Error: teardown error + + 5 | + 6 | test.afterEach(() => { + > 7 | throw new Error('teardown error'); + | ^ + 8 | }); + 9 | + at ${test.info().outputPath('a.test.ts')}:7:17 + + + 1 interrupted + a.test.ts:3:13 › foo ─────────────────────────────────────────────────────────────────────────── +`); + }); + + test('pause on error', async ({ interactWithTestRunner }) => { + const runner = await interactWithTestRunner({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', async ({}) => { + expect.soft(2).toBe(3); + expect(3).toBe(4); + }); + `, + }, { pause: true, reporter: 'line' }, { PW_TEST_DEBUG_REPORTERS: '1' }); + + await runner.waitForOutput('Paused on error. Press Ctrl+C to end.'); + const { exitCode } = await runner.kill('SIGINT'); + expect(exitCode).toBe(130); + + expect(stripAnsi(runner.output)).toEqual(` +Running 1 test using 1 worker + +[1/1] a.test.ts:3:13 › fails + 1) a.test.ts:3:13 › fails ──────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 3 + Received: 2 + + 2 | import { test, expect } from '@playwright/test'; + 3 | test('fails', async ({}) => { + > 4 | expect.soft(2).toBe(3); + | ^ + 5 | expect(3).toBe(4); + 6 | }); + 7 | + at ${test.info().outputPath('a.test.ts')}:4:26 + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 4 + Received: 3 + + 3 | test('fails', async ({}) => { + 4 | expect.soft(2).toBe(3); + > 5 | expect(3).toBe(4); + | ^ + 6 | }); + 7 | + at ${test.info().outputPath('a.test.ts')}:5:21 + + Paused on error. Press Ctrl+C to end. + + +[1/1] a.test.ts:3:13 › fails + + + 1 failed + a.test.ts:3:13 › fails ───────────────────────────────────────────────────────────────────────── +`); + }); +});