From 716cf30752347d5f2a410871261c0c5d56c9258b Mon Sep 17 00:00:00 2001 From: Yaroslav Serhieiev Date: Tue, 11 Oct 2022 16:19:28 +0300 Subject: [PATCH] feat(jest): new listener events (#3626) --- detox/index.d.ts | 18 +- detox/internals.d.ts | 50 ++++-- detox/local-cli/init.js | 2 +- detox/local-cli/test.test.js | 72 ++++++-- .../testCommand/TestRunnerCommand.js | 13 +- detox/runners/jest/reporters/DetoxReporter.js | 24 ++- detox/runners/jest/testEnvironment/index.js | 161 +++++++++--------- .../listeners/DetoxCoreListener.js | 26 +-- .../listeners/DetoxPlatformFilterListener.js | 2 +- .../src/configuration/composeRunnerConfig.js | 4 +- .../configuration/composeRunnerConfig.test.js | 28 ++- detox/src/ipc/IPCClient.js | 10 +- detox/src/ipc/IPCServer.js | 28 +-- detox/src/ipc/SessionState.js | 6 +- detox/src/realms/DetoxContext.js | 4 +- detox/src/realms/DetoxInternalsFacade.js | 2 +- detox/src/realms/DetoxPrimaryContext.js | 7 +- detox/src/realms/DetoxSecondaryContext.js | 4 +- detox/src/symbols.js | 4 +- detox/src/utils/Timer.js | 93 +++++----- detox/src/utils/Timer.test.js | 83 ++++----- detox/src/utils/errorUtils.js | 20 +++ .../detox-init-timeout/testEnvironment.js | 2 +- detox/test/e2e/utils/timeoutUtils.js | 6 - detox/test/types/detox-internals-tests.ts | 18 ++ docs/api/internals.md | 13 +- docs/config/testRunner.mdx | 19 ++- docs/guide/migration.md | 6 +- docs/guide/proguard-configuration.md | 1 + 29 files changed, 439 insertions(+), 287 deletions(-) delete mode 100644 detox/test/e2e/utils/timeoutUtils.js diff --git a/detox/index.d.ts b/detox/index.d.ts index ce2edd73c4..964f9b8ba2 100644 --- a/detox/index.d.ts +++ b/detox/index.d.ts @@ -139,9 +139,13 @@ declare global { */ jest?: { /** - * Device init timeout + * Environment setup timeout */ - initTimeout?: number | undefined; + setupTimeout?: number | undefined; + /** + * Environment teardown timeout + */ + teardownTimeout?: number | undefined; /** * Insist on CLI-based retry mechanism even when the failed tests have been handled * by jest.retryTimes(n) mechanism from Jest Circus. @@ -154,6 +158,16 @@ declare global { * Retries count. Zero means a single attempt to run tests. */ retries?: number; + /** + * When true, tells Detox CLI to cancel next retrying if it gets + * at least one report about a permanent test suite failure. + * Has no effect, if {@link DetoxTestRunnerConfig#retries} is + * undefined or set to zero. + * + * @default false + * @see {DetoxInternals.DetoxTestFileReport#isPermanentFailure} + */ + bail?: boolean; /** * Custom handler to process --inspect-brk CLI flag. * Use it when you rely on another test runner than Jest. diff --git a/detox/internals.d.ts b/detox/internals.d.ts index 8cb215b504..c35e4d540a 100644 --- a/detox/internals.d.ts +++ b/detox/internals.d.ts @@ -54,14 +54,13 @@ declare global { onRunFinish(event: unknown): Promise; /** - * Reports to Detox CLI about failed tests that could have been re-run if + * Reports to Detox CLI about passed and failed test files. + * The failed test files might be re-run again if * {@link Detox.DetoxTestRunnerConfig#retries} is set to a non-zero. * - * @param testFilePaths array of failed test files' paths - * @param permanent whether the failure is permanent, and the tests - * should not be re-run. + * @param testResults - reports about test files */ - reportFailedTests(testFilePaths: string[], permanent?: boolean): Promise; + reportTestResults(testResults: DetoxTestFileReport[]): Promise; // endregion readonly config: RuntimeConfig; @@ -92,7 +91,7 @@ declare global { testRunnerArgv: Record; override: Partial; /** @inheritDoc */ - global: NodeJS.Global; + global: NodeJS.Global | {}; /** * Worker ID. Used to distinguish allocated workers in parallel test execution environment. * @@ -111,7 +110,7 @@ declare global { * {@link DetoxInternals.Facade#setup} might override {@link Console} methods * to integrate it with Detox loggeing subsystem. */ - global: NodeJS.Global; + global: NodeJS.Global | {}; /** * Worker ID. Used to distinguish allocated workers in parallel test execution environment. * @@ -120,25 +119,48 @@ declare global { workerId: string; }; + type DetoxTestFileReport = { + /** + * Global or relative path to the failed test file. + */ + testFilePath: string; + /** + * Whether the test passed or not. + */ + success: boolean; + /** + * Top-level error if the entire test file failed. + */ + testExecError?: { name?: string; message: string; stack?: string; }; + /** + * If the test failed, it should tell whether the failure is permanent. + * Permanent failure means that the test file should not be re-run. + * + * @default false + * @see {Detox.DetoxTestRunnerConfig#retries} + */ + isPermanentFailure?: boolean; + }; + type SessionState = Readonly<{ /** * Randomly generated ID for the entire Detox test session, including retries. */ id: string; /** - * Permanently failed test file paths. + * Results of test file executions. Primarily used for Detox CLI retry mechanism. */ - failedTestFiles: string[]; - /** - * Failed test file paths suggested to retry via Detox CLI mechanism. - */ - testFilesToRetry: string[]; + testResults: DetoxTestFileReport[]; /** * Retry index of the test session: 0..retriesCount. */ testSessionIndex: number; /** - * TODO + * Count of Detox contexts with a worker installed. + * Oversimplified, it reflects the count of allocated devices in the current test session. + * + * @see {Facade#init} + * @see {Facade#installWorker} */ workersCount: number; }>; diff --git a/detox/local-cli/init.js b/detox/local-cli/init.js index ee530d9eab..35aea5751a 100644 --- a/detox/local-cli/init.js +++ b/detox/local-cli/init.js @@ -76,7 +76,7 @@ function createDefaultConfigurations() { config: 'e2e/jest.config.js', }, jest: { - initTimeout: 120000, + setupTimeout: 120000, }, }, apps: { diff --git a/detox/local-cli/test.test.js b/detox/local-cli/test.test.js index 948b51ab51..e554d15ae4 100644 --- a/detox/local-cli/test.test.js +++ b/detox/local-cli/test.test.js @@ -8,6 +8,8 @@ jest.mock('../src/devices/DeviceRegistry'); jest.mock('../src/devices/allocation/drivers/android/genycloud/GenyDeviceRegistryFactory'); jest.mock('./utils/jestInternals'); +const cp = require('child_process'); +const cpSpawn = cp.spawn; const os = require('os'); const path = require('path'); @@ -28,6 +30,10 @@ describe('CLI', () => { let GenyDeviceRegistryFactory; let jestInternals; + afterEach(() => { + cp.spawn = cpSpawn; + }); + beforeEach(() => { _cliCallDump = undefined; _env = process.env; @@ -139,11 +145,28 @@ describe('CLI', () => { }); test.each([['-R'], ['--retries']])('%s should execute unsuccessful run N extra times', async (__retries) => { + function toTestResult(testFilePath) { + return { + testFilePath, + success: false, + isPermanentFailure: false, + }; + } + const context = require('../internals'); - context.session.testFilesToRetry = ['e2e/failing1.test.js', 'e2e/failing2.test.js']; - context.session.testFilesToRetry.splice = jest.fn() - .mockImplementationOnce(() => ['e2e/failing1.test.js', 'e2e/failing2.test.js']) - .mockImplementationOnce(() => ['e2e/failing2.test.js']); + + jest.spyOn(cp, 'spawn') + .mockImplementationOnce((...args) => { + context.session.testResults = ['e2e/failing1.test.js', 'e2e/failing2.test.js'].map(toTestResult); + return cpSpawn(...args); + }) + .mockImplementationOnce((...args) => { + context.session.testResults = ['e2e/failing2.test.js'].map(toTestResult); + return cpSpawn(...args); + }) + .mockImplementationOnce((...args) => { + return cpSpawn(...args); + }); mockExitCode(1); @@ -154,18 +177,35 @@ describe('CLI', () => { expect(cliCall(2).argv).toEqual([expect.stringMatching(/executable$/), '--config', 'e2e/config.json', 'e2e/failing2.test.js']); }); - test.each([['-R'], ['--retries']])('%s should bail if has permanently failed tests', async (__retries) => { - const context = require('../internals'); - context.session.failedTestFiles = ['e2e/failing1.test.js']; - context.session.testFilesToRetry = ['e2e/failing2.test.js']; + describe('when there are permanently failed tests', () => { + beforeEach(() => { + const context = require('../internals'); + context.session.testResults = ['e2e/failing1.test.js', 'e2e/failing2.test.js'].map((testFilePath, index) => ({ + testFilePath, + success: false, + isPermanentFailure: index > 0, + })); - mockExitCode(1); + mockExitCode(1); + }); - await run(__retries, 2).catch(_.noop); + test.each([['-R'], ['--retries']])('%s should not bail by default', async (__retries) => { + await run(__retries, 2).catch(_.noop); - expect(cliCall(0).env).not.toHaveProperty('DETOX_RERUN_INDEX'); - expect(cliCall(0).argv).toEqual([expect.stringMatching(/executable$/), '--config', 'e2e/config.json']); - expect(cliCall(1)).toBe(null); + expect(cliCall(0).env).not.toHaveProperty('DETOX_RERUN_INDEX'); + expect(cliCall(0).argv).toEqual([expect.stringMatching(/executable$/), '--config', 'e2e/config.json']); + expect(cliCall(1).argv).toEqual([expect.stringMatching(/executable$/), '--config', 'e2e/config.json', 'e2e/failing1.test.js']); + // note that it does not take the permanently failed test + }); + + test.each([['-R'], ['--retries']])('%s should bail if configured', async (__retries) => { + detoxConfig.testRunner.bail = true; + await run(__retries, 2).catch(_.noop); + + expect(cliCall(0).env).not.toHaveProperty('DETOX_RERUN_INDEX'); + expect(cliCall(0).argv).toEqual([expect.stringMatching(/executable$/), '--config', 'e2e/config.json']); + expect(cliCall(1)).toBe(null); + }); }); test.each([['-R'], ['--retries']])('%s should not restart test runner if there are no failing tests paths', async (__retries) => { @@ -178,7 +218,11 @@ describe('CLI', () => { test.each([['-R'], ['--retries']])('%s should retain -- <...explicitPassthroughArgs>', async (__retries) => { const context = require('../internals'); - context.session.testFilesToRetry = ['tests/failing.test.js']; + context.session.testResults = [{ + testFilePath: 'tests/failing.test.js', + success: false, + isPermanentFailure: false, + }]; mockExitCode(1); diff --git a/detox/local-cli/testCommand/TestRunnerCommand.js b/detox/local-cli/testCommand/TestRunnerCommand.js index a1f33f12d1..4ebf44c05d 100644 --- a/detox/local-cli/testCommand/TestRunnerCommand.js +++ b/detox/local-cli/testCommand/TestRunnerCommand.js @@ -99,13 +99,20 @@ class TestRunnerCommand { } catch (e) { launchError = e; - const { failedTestFiles, testFilesToRetry } = detox.session; - if (!_.isEmpty(failedTestFiles) || _.isEmpty(testFilesToRetry)) { + const failedTestFiles = detox.session.testResults.filter(r => !r.success); + + const { bail } = detox.config.testRunner; + if (bail && failedTestFiles.some(r => r.isPermanentFailure)) { + throw e; + } + + const testFilesToRetry = failedTestFiles.filter(r => !r.isPermanentFailure).map(r => r.testFilePath); + if (_.isEmpty(testFilesToRetry)) { throw e; } if (--runsLeft > 0) { - this._argv._ = testFilesToRetry.splice(0, Infinity); + this._argv._ = testFilesToRetry; // @ts-ignore detox.session.testSessionIndex++; // it is always a primary context, so we can update it } diff --git a/detox/runners/jest/reporters/DetoxReporter.js b/detox/runners/jest/reporters/DetoxReporter.js index 7705b05175..a2ec1a0f15 100644 --- a/detox/runners/jest/reporters/DetoxReporter.js +++ b/detox/runners/jest/reporters/DetoxReporter.js @@ -1,5 +1,25 @@ -const { VerboseReporter: JestVerboseReporter } = require('@jest/reporters'); // eslint-disable-line node/no-extraneous-require +const resolveFrom = require('resolve-from'); +/** @type {typeof import('@jest/reporters').VerboseReporter} */ +const JestVerboseReporter = require(resolveFrom(process.cwd(), '@jest/reporters')).VerboseReporter; -class DetoxReporter extends JestVerboseReporter {} +const { config, reportTestResults } = require('../../../internals'); + +class DetoxReporter extends JestVerboseReporter { + /** + * @param {import('@jest/test-result').TestResult} testResult + */ + async onTestResult(test, testResult, results) { + await super.onTestResult(test, testResult, results); + + await reportTestResults([{ + success: !testResult.failureMessage, + testFilePath: testResult.testFilePath, + testExecError: testResult.testExecError, + isPermanentFailure: config.testRunner.jest.retryAfterCircusRetries + ? false + : testResult.testResults.some(r => r.invocations > 1) + }]); + } +} module.exports = DetoxReporter; diff --git a/detox/runners/jest/testEnvironment/index.js b/detox/runners/jest/testEnvironment/index.js index 22405bf91b..35bc76c4a5 100644 --- a/detox/runners/jest/testEnvironment/index.js +++ b/detox/runners/jest/testEnvironment/index.js @@ -1,6 +1,6 @@ const resolveFrom = require('resolve-from'); const maybeNodeEnvironment = require(resolveFrom(process.cwd(), 'jest-environment-node')); -// @ts-ignore +/** @type {typeof import('@jest/environment').JestEnvironment} */ const NodeEnvironment = maybeNodeEnvironment.default || maybeNodeEnvironment; const detox = require('../../../internals'); @@ -33,110 +33,94 @@ class DetoxCircusEnvironment extends NodeEnvironment { constructor(config, context) { super(assertJestCircus27(config), assertExistingContext(context)); - /** @private */ - this._timer = null; - /** @private */ - this._listenerFactories = { - DetoxInitErrorListener, - DetoxPlatformFilterListener, - DetoxCoreListener, - SpecReporter, - WorkerAssignReporter, - }; /** @private */ this._shouldManageDetox = detox.getStatus() === 'inactive'; /** @private */ - this._setupFailed = false; + this._timer = new Timer(); + /** @internal */ this.testPath = context.testPath; /** @protected */ this.testEventListeners = []; /** @protected */ - this.initTimeout = detox.config.testRunner.jest.initTimeout; - /** @internal */ + this.setupTimeout = detox.config.testRunner.jest.setupTimeout; + /** @protected */ + this.teardownTimeout = detox.config.testRunner.jest.teardownTimeout; + log.trace.begin(this.testPath); - this.setup = log.trace.complete.bind(null, 'set up environment', this.setup.bind(this)); + const _setup = this.setup.bind(this); + this.setup = async () => { + await log.trace.complete('set up environment', async () => { + try { + this._timer.schedule(this.setupTimeout); + await this._timer.run(`setting up Detox environment`, _setup); + } catch (error) { + this._timer.schedule(this.teardownTimeout); + await this._handleTestEventAsync({ name: 'environment_setup_failure', error }); + throw error; + } finally { + this._timer.clear(); + } + }); + }; + const _teardown = this.teardown.bind(this); this.teardown = async () => { - try { - await log.trace.complete('tear down environment', _teardown); - } finally { - await log.trace.end(); - } + await log.trace.complete('tear down environment', async () => { + try { + this._timer.schedule(this.teardownTimeout); + await this._timer.run(`tearing down Detox environment`, _teardown); + } catch (error) { + if (this._timer.expired) { + this._timer.schedule(this.teardownTimeout); + } + + await this._handleTestEventAsync({ name: 'environment_teardown_failure', error }); + throw error; + } finally { + this._timer.clear(); + log.trace.end(); + } + }); }; + + this.registerListeners({ + DetoxInitErrorListener, + DetoxPlatformFilterListener, + DetoxCoreListener, + SpecReporter, + WorkerAssignReporter, + }); } /** @override */ async setup() { - try { - await super.setup(); - await Timer.run({ - description: `setting up Detox environment`, - timeout: this.initTimeout, - fn: async () => { - await this.initDetox(); - this._instantiateListeners(); - }, - }); - } catch (e) { - this._setupFailed = true; - throw e; - } + await this.initDetox(); } - /** @override */ - async handleTestEvent(event, state) { - const { name } = event; + handleTestEvent = async (event, state) => { + this._timer.schedule(state.testTimeout != null ? state.testTimeout : this.setupTimeout); - if (SYNC_CIRCUS_EVENTS.has(name)) { - return this._handleTestEventSync(event, state); - } - - this._timer = new Timer({ - description: `handling jest-circus "${name}" event`, - timeout: state.testTimeout != null ? state.testTimeout : this.initTimeout, - }); - - try { - for (const listener of this.testEventListeners) { - if (typeof listener[name] !== 'function') { - continue; - } - - try { - await this._timer.run(() => listener[name](event, state)); - } catch (listenerError) { - log.error(listenerError); - break; - } - } - } finally { - this._timer.dispose(); - this._timer = null; + if (SYNC_CIRCUS_EVENTS.has(event.name)) { + this._handleTestEventSync(event, state); + } else { + await this._handleTestEventAsync(event, state); } - } + }; /** @override */ async teardown() { - await Timer.run({ - description: `tearing down Detox environment`, - timeout: this.initTimeout, - fn: async () => { - try { - if (this._setupFailed) { - await detox.reportFailedTests([this.testPath], false); - } - } finally { - await this.cleanupDetox(); - } - }, - }); + await this.cleanupDetox(); } /** @protected */ registerListeners(map) { - Object.assign(this._listenerFactories, map); + for (const Listener of Object.values(map)) { + this.testEventListeners.push(new Listener({ + env: this, + })); + } } /** @@ -153,6 +137,8 @@ class DetoxCircusEnvironment extends NodeEnvironment { } else { await detox.installWorker(opts); } + + return detox.worker; } /** @protected */ @@ -176,11 +162,22 @@ class DetoxCircusEnvironment extends NodeEnvironment { } /** @private */ - _instantiateListeners() { - for (const Listener of Object.values(this._listenerFactories)) { - this.testEventListeners.push(new Listener({ - env: this, - })); + async _handleTestEventAsync(event, state = null) { + const description = `handling ${state ? 'jest-circus' : 'jest-environment'} "${event.name}" event`; + + for (const listener of this.testEventListeners) { + if (typeof listener[event.name] !== 'function') { + continue; + } + + try { + await this._timer.run(description, () => listener[event.name](event, state)); + } catch (listenerError) { + log.error(listenerError); + if (this._timer.expired) { + break; + } + } } } } diff --git a/detox/runners/jest/testEnvironment/listeners/DetoxCoreListener.js b/detox/runners/jest/testEnvironment/listeners/DetoxCoreListener.js index af94a9d465..a4ea10635d 100644 --- a/detox/runners/jest/testEnvironment/listeners/DetoxCoreListener.js +++ b/detox/runners/jest/testEnvironment/listeners/DetoxCoreListener.js @@ -14,7 +14,7 @@ class DetoxCoreListener { this._startedTests = new WeakSet(); this._testsFailedBeforeStart = new WeakSet(); this._env = env; - this._testRunTimes = 1; + this._circusRetryTimes = 1; } async setup() { @@ -46,7 +46,7 @@ class DetoxCoreListener { } const circusRetryTimes = +this._env.global[RETRY_TIMES]; - this._testRunTimes = isNaN(circusRetryTimes) ? 1 : 1 + circusRetryTimes; + this._circusRetryTimes = isNaN(circusRetryTimes) ? 1 : 1 + circusRetryTimes; } async hook_start(event, state) { @@ -94,14 +94,6 @@ class DetoxCoreListener { } } - async run_finish(_event, state) { - const hasFailedTests = this._hasFailedTests(state.rootDescribeBlock); - if (hasFailedTests) { - const handledByJestCircus = this._testRunTimes > 1 && !detoxInternals.config.testRunner.jest.retryAfterCircusRetries; - await detoxInternals.reportFailedTests([this._env.testPath], handledByJestCircus); - } - } - async _onBeforeActualTestStart(test) { if (!test || test.status === 'skip' || this._startedTests.has(test) || this._testsFailedBeforeStart.has(test)) { return false; @@ -127,19 +119,7 @@ class DetoxCoreListener { _getTestInvocations(test) { const { testSessionIndex } = detoxInternals.session; - return testSessionIndex * this._testRunTimes + test.invocations; - } - - _hasFailedTests(block) { - if (block.children) { - for (const child of block.children) { - if (this._hasFailedTests(child)) { - return true; - } - } - } - - return block.errors ? block.errors.length > 0 : false; + return testSessionIndex * this._circusRetryTimes + test.invocations; } _getTestMetadata(test) { diff --git a/detox/runners/jest/testEnvironment/listeners/DetoxPlatformFilterListener.js b/detox/runners/jest/testEnvironment/listeners/DetoxPlatformFilterListener.js index 45ac2d7e22..7af0fd1726 100644 --- a/detox/runners/jest/testEnvironment/listeners/DetoxPlatformFilterListener.js +++ b/detox/runners/jest/testEnvironment/listeners/DetoxPlatformFilterListener.js @@ -5,7 +5,7 @@ const { device } = require('../../../..'); const PLATFORM_REGEXP = /^:([^:]+):/; class DetoxPlatformFilterListener { - constructor() { + setup() { this._platform = device.getPlatform(); } diff --git a/detox/src/configuration/composeRunnerConfig.js b/detox/src/configuration/composeRunnerConfig.js index f712add407..54a5f674c8 100644 --- a/detox/src/configuration/composeRunnerConfig.js +++ b/detox/src/configuration/composeRunnerConfig.js @@ -32,8 +32,10 @@ function composeRunnerConfig(opts) { retries: 0, inspectBrk: inspectBrkHookDefault, forwardEnv: false, + bail: false, jest: { - initTimeout: 300000, + setupTimeout: 300000, + teardownTimeout: 30000, retryAfterCircusRetries: false, reportSpecs: undefined, reportWorkerAssign: true, diff --git a/detox/src/configuration/composeRunnerConfig.test.js b/detox/src/configuration/composeRunnerConfig.test.js index 673e9b64da..609e6c5c0c 100644 --- a/detox/src/configuration/composeRunnerConfig.test.js +++ b/detox/src/configuration/composeRunnerConfig.test.js @@ -38,12 +38,14 @@ describe('composeRunnerConfig', () => { _: [], }, jest: { - initTimeout: 300000, + setupTimeout: 300000, + teardownTimeout: 30000, retryAfterCircusRetries: false, reportSpecs: undefined, reportWorkerAssign: true, }, retries: 0, + bail: false, inspectBrk: false, forwardEnv: false, }); @@ -53,10 +55,11 @@ describe('composeRunnerConfig', () => { globalConfig.testRunner = { args: { $0: 'nyc jest' }, jest: { - initTimeout: 5000, + setupTimeout: 5000, retryAfterCircusRetries: true, reportSpecs: false, }, + bail: true, retries: 1, inspectBrk: true, forwardEnv: true, @@ -68,11 +71,13 @@ describe('composeRunnerConfig', () => { _: [], }, jest: { - initTimeout: 5000, + setupTimeout: 5000, + teardownTimeout: 30000, retryAfterCircusRetries: true, reportSpecs: false, reportWorkerAssign: true, }, + bail: true, retries: 1, inspectBrk: true, forwardEnv: true, @@ -83,10 +88,12 @@ describe('composeRunnerConfig', () => { localConfig.testRunner = { args: { $0: 'nyc jest' }, jest: { - initTimeout: 120000, + setupTimeout: 120000, + teardownTimeout: 30000, retryAfterCircusRetries: true, reportSpecs: true, }, + bail: true, retries: 1, inspectBrk: true, forwardEnv: true, @@ -98,11 +105,13 @@ describe('composeRunnerConfig', () => { _: [], }, jest: { - initTimeout: 120000, + setupTimeout: 120000, + teardownTimeout: 30000, retryAfterCircusRetries: true, reportSpecs: true, reportWorkerAssign: true, }, + bail: true, retries: 1, inspectBrk: true, forwardEnv: true, @@ -147,7 +156,8 @@ describe('composeRunnerConfig', () => { expect(composeRunnerConfig()).toEqual(expect.objectContaining({ jest: { customProperty: 1, - initTimeout: 300000, + setupTimeout: 300000, + teardownTimeout: 30000, otherProperty: true, retryAfterCircusRetries: false, reportSpecs: true, @@ -179,6 +189,7 @@ describe('composeRunnerConfig', () => { jest: { reportSpecs: true, }, + bail: true, retries: 1, }; @@ -192,6 +203,7 @@ describe('composeRunnerConfig', () => { jest: { reportSpecs: false, }, + bail: false, retries: 3, }; @@ -205,11 +217,13 @@ describe('composeRunnerConfig', () => { _: ['second.test.js'], }, jest: { - initTimeout: 300_000, + setupTimeout: 300000, + teardownTimeout: 30000, retryAfterCircusRetries: false, reportSpecs: false, reportWorkerAssign: true, }, + bail: false, retries: 3, inspectBrk: false, forwardEnv: false, diff --git a/detox/src/ipc/IPCClient.js b/detox/src/ipc/IPCClient.js index d94c3cbc5b..fae7d26645 100644 --- a/detox/src/ipc/IPCClient.js +++ b/detox/src/ipc/IPCClient.js @@ -1,6 +1,7 @@ const { IPC } = require('node-ipc'); const { DetoxInternalError } = require('../errors'); +const { serializeObjectWithError } = require('../utils/errorUtils'); class IPCClient { constructor({ id, logger, state }) { @@ -51,11 +52,12 @@ class IPCClient { } /** - * @param {string[]} testFilePaths - * @param {Boolean} permanent + * @param {DetoxInternals.DetoxTestFileReport[]} testResults */ - async reportFailedTests(testFilePaths, permanent) { - await this._emit('failedTests', { testFilePaths, permanent }); + async reportTestResults(testResults) { + await this._emit('reportTestResults', { + testResults: testResults.map(r => serializeObjectWithError(r, 'testExecError')), + }); } async _connectToServer() { diff --git a/detox/src/ipc/IPCServer.js b/detox/src/ipc/IPCServer.js index fa35c46da3..7364caa009 100644 --- a/detox/src/ipc/IPCServer.js +++ b/detox/src/ipc/IPCServer.js @@ -1,5 +1,8 @@ +const { uniqBy } = require('lodash'); const { IPC } = require('node-ipc'); +const { serializeObjectWithError } = require('../utils/errorUtils'); + class IPCServer { /** * @param {object} options @@ -27,12 +30,12 @@ class IPCServer { this._ipc.config.appspace = 'detox.'; this._ipc.config.logger = (msg) => this._logger.trace(msg); - await new Promise((resolve) => { + await new Promise((resolve) => { // TODO: handle reject this._ipc.serve(() => resolve()); this._ipc.server.on('registerContext', this.onRegisterContext.bind(this)); this._ipc.server.on('registerWorker', this.onRegisterWorker.bind(this)); - this._ipc.server.on('failedTests', this.onFailedTests.bind(this)); + this._ipc.server.on('reportTestResults', this.onReportTestResults.bind(this)); this._ipc.server.start(); }); } @@ -53,8 +56,7 @@ class IPCServer { this._sessionState.contexts.push(id); this._ipc.server.emit(socket, 'registerContextDone', { - failedTestFiles: this._sessionState.failedTestFiles, - testFilesToRetry: this._sessionState.testFilesToRetry, + testResults: this._sessionState.testResults, testSessionIndex: this._sessionState.testSessionIndex, }); } @@ -73,20 +75,20 @@ class IPCServer { } } - onFailedTests({ testFilePaths, permanent }, socket = null) { - if (permanent) { - this._sessionState.failedTestFiles.push(...testFilePaths); - } else { - this._sessionState.testFilesToRetry.push(...testFilePaths); - } + onReportTestResults({ testResults }, socket = null) { + const merged = uniqBy([ + ...testResults.map(r => serializeObjectWithError(r, 'testExecError')), + ...this._sessionState.testResults + ], 'testFilePath'); + + this._sessionState.testResults.splice(0, Infinity, ...merged); if (socket) { - this._ipc.server.emit(socket, 'failedTestsDone', {}); + this._ipc.server.emit(socket, 'reportTestResultsDone', {}); } this._ipc.server.broadcast('sessionStateUpdate', { - failedTestFiles: this._sessionState.failedTestFiles, - testFilesToRetry: this._sessionState.testFilesToRetry, + testResults: this._sessionState.testResults, }); } } diff --git a/detox/src/ipc/SessionState.js b/detox/src/ipc/SessionState.js index d52127cf2e..56dfedec89 100644 --- a/detox/src/ipc/SessionState.js +++ b/detox/src/ipc/SessionState.js @@ -9,8 +9,7 @@ class SessionState { detoxConfigSnapshotPath = '', detoxConfig = null, detoxIPCServer = '', - failedTestFiles = [], - testFilesToRetry = [], + testResults = [], testSessionIndex = 0, workersCount = 0 }) { @@ -19,8 +18,7 @@ class SessionState { this.detoxConfigSnapshotPath = detoxConfigSnapshotPath; this.detoxConfig = detoxConfig; this.detoxIPCServer = detoxIPCServer; - this.failedTestFiles = failedTestFiles; - this.testFilesToRetry = testFilesToRetry; + this.testResults = testResults; this.testSessionIndex = testSessionIndex; this.workersCount = workersCount; } diff --git a/detox/src/realms/DetoxContext.js b/detox/src/realms/DetoxContext.js index 83fe5db60c..5860d749e1 100644 --- a/detox/src/realms/DetoxContext.js +++ b/detox/src/realms/DetoxContext.js @@ -36,7 +36,7 @@ class DetoxContext { }; this[symbols.getStatus] = this[symbols.getStatus].bind(this); - this[symbols.reportFailedTests] = this[symbols.reportFailedTests].bind(this); + this[symbols.reportTestResults] = this[symbols.reportTestResults].bind(this); this[symbols.resolveConfig] = this[symbols.resolveConfig].bind(this); this[symbols.installWorker] = this[symbols.installWorker].bind(this); this[symbols.uninstallWorker] = this[symbols.uninstallWorker].bind(this); @@ -103,7 +103,7 @@ class DetoxContext { }, }); /** @abstract */ - [symbols.reportFailedTests](_testFilePaths, _permanent) {} + [symbols.reportTestResults](_testResults) {} /** * @abstract * @param {Partial} _opts diff --git a/detox/src/realms/DetoxInternalsFacade.js b/detox/src/realms/DetoxInternalsFacade.js index ab9d9a20e4..fb18392054 100644 --- a/detox/src/realms/DetoxInternalsFacade.js +++ b/detox/src/realms/DetoxInternalsFacade.js @@ -26,7 +26,7 @@ class DetoxInternalsFacade { this.onTestFnStart = context[symbols.onTestFnStart]; this.onTestFnSuccess = context[symbols.onTestFnSuccess]; this.onTestStart = context[symbols.onTestStart]; - this.reportFailedTests = context[symbols.reportFailedTests]; + this.reportTestResults = context[symbols.reportTestResults]; this.resolveConfig = context[symbols.resolveConfig]; this.session = context[symbols.session]; this.tracing = context[symbols.tracing]; diff --git a/detox/src/realms/DetoxPrimaryContext.js b/detox/src/realms/DetoxPrimaryContext.js index 8a9ca6071d..6b0203938e 100644 --- a/detox/src/realms/DetoxPrimaryContext.js +++ b/detox/src/realms/DetoxPrimaryContext.js @@ -46,9 +46,9 @@ class DetoxPrimaryContext extends DetoxContext { } //#region Internal members - async [symbols.reportFailedTests](testFilePaths, permanent = false) { + async [symbols.reportTestResults](testResults) { if (this[_ipcServer]) { - this[_ipcServer].onFailedTests({ testFilePaths, permanent }); + this[_ipcServer].onReportTestResults({ testResults }); } } @@ -286,8 +286,7 @@ class DetoxPrimaryContext extends DetoxContext { return true; } - const { failedTestFiles, testFilesToRetry } = this[$sessionState]; - return failedTestFiles.length + testFilesToRetry.length > 0; + return this[$sessionState].testResults.some(r => !r.success); } async[_resetLockFile]() { diff --git a/detox/src/realms/DetoxSecondaryContext.js b/detox/src/realms/DetoxSecondaryContext.js index 92b4f4267d..b5acf89c3f 100644 --- a/detox/src/realms/DetoxSecondaryContext.js +++ b/detox/src/realms/DetoxSecondaryContext.js @@ -29,9 +29,9 @@ class DetoxSecondaryContext extends DetoxContext { } //#region Internal members - async [symbols.reportFailedTests](testFilePaths, permanent = false) { + async [symbols.reportTestResults](testResults) { if (this[_ipcClient]) { - await this[_ipcClient].reportFailedTests(testFilePaths, permanent); + await this[_ipcClient].reportTestResults(testResults); } else { throw new DetoxInternalError('Detected an attempt to report failed tests using a non-initialized context.'); } diff --git a/detox/src/symbols.js b/detox/src/symbols.js index 21db414e6e..091cee48e1 100644 --- a/detox/src/symbols.js +++ b/detox/src/symbols.js @@ -18,7 +18,7 @@ * readonly onTestFnStart: unique symbol; * readonly onTestFnSuccess: unique symbol; * readonly onTestStart: unique symbol; - * readonly reportFailedTests: unique symbol; + * readonly reportTestResults: unique symbol; * readonly resolveConfig: unique symbol; * readonly session: unique symbol; * readonly tracing: unique symbol; @@ -43,7 +43,7 @@ module.exports = { //#endregion //#region IPC - reportFailedTests: Symbol('reportFailedTests'), + reportTestResults: Symbol('reportTestResults'), //#endregion //#region Main diff --git a/detox/src/utils/Timer.js b/detox/src/utils/Timer.js index f545767312..8bc114aa80 100644 --- a/detox/src/utils/Timer.js +++ b/detox/src/utils/Timer.js @@ -1,65 +1,82 @@ -// @ts-nocheck -const DetoxRuntimeError = require('../errors/DetoxRuntimeError'); +const { DetoxRuntimeError, DetoxInternalError } = require('../errors'); const Deferred = require('./Deferred'); class Timer { + constructor() { + /** @private */ + this._eta = NaN; + /** @private */ + this._timeout = NaN; + /** @type {NodeJS.Timer | null} */ + this._timeoutHandle = null; + /** @type {Deferred | null} */ + this._timeoutDeferred = null; + } + + clear() { + if (this._timeoutHandle) { + clearTimeout(this._timeoutHandle); + this._timeoutHandle = null; + } + + this._eta = NaN; + this._timeout = NaN; + this._timeoutDeferred = null; + } + + get expired() { + return this._timeoutDeferred ? this._timeoutDeferred.isResolved() : false; + } + /** - * @param {string} description - gives more context for thrown errors * @param {number} timeout - maximal allowed duration in milliseconds */ - constructor({ description, timeout }) { - /** @private */ + schedule(timeout) { + this.clear(); + + this._eta = Date.now() + timeout; this._timeout = timeout; - /** @public */ - this.description = description; + this._timeoutDeferred = new Deferred(); + this._timeoutHandle = setTimeout(() => { + this._timeoutDeferred.resolve(); + }, this._timeout); - this._schedule(); + return this; } - _schedule() { - this._createdAt = Date.now(); - this._timeoutDeferred = new Deferred(); + extend(ms) { + if (this.expired) { + return this.schedule(ms); + } + + clearTimeout(this._timeoutHandle); + this._eta += ms; + this._timeout += ms; this._timeoutHandle = setTimeout(() => { this._timeoutDeferred.resolve(); - }, this._timeout); + }, this._eta - Date.now()); } - async run(action) { + async run(description, action) { + if (!this._timeoutDeferred) { + throw new DetoxInternalError('Cannot run a timer action from an uninitialized timer'); + } + const error = new DetoxRuntimeError({ - message: `Exceeded timeout of ${this._timeout}ms while ${this.description}`, + message: `Exceeded timeout of ${this._timeout}ms while ${description}`, noStack: true, }); + if (this.expired) { + throw error; + } + return Promise.race([ this._timeoutDeferred.promise.then(() => { throw error; }), Promise.resolve().then(action), ]); } - - dispose() { - clearTimeout(this._timeoutHandle); - } - - reset(extraDelayMs) { - this._timeout = extraDelayMs; - - if (this._timeoutDeferred.status !== Deferred.PENDING) { - this._schedule(); - } else { - clearTimeout(this._timeoutHandle); - this._timeoutHandle = setTimeout(() => this._timeoutDeferred.resolve(), extraDelayMs); - } - } - - static async run({ description, timeout, fn }) { - const timer = new Timer({ description, timeout }); - try { - await timer.run(fn); - } finally { - timer.dispose(); - } - } } module.exports = Timer; diff --git a/detox/src/utils/Timer.test.js b/detox/src/utils/Timer.test.js index b18fb22005..de4493a3dc 100644 --- a/detox/src/utils/Timer.test.js +++ b/detox/src/utils/Timer.test.js @@ -1,90 +1,75 @@ jest.useFakeTimers('modern'); -describe('Timer', () => { - let Timer; +const Deferred = require('./Deferred'); +const Timer = require('./Timer'); - beforeEach(() => { - Timer = require('./Timer'); +describe('Timer', () => { + it('should throw on attempt to run when uninitialized', async () => { + await expect(new Timer().run('', () => {})).rejects.toThrow(/Cannot run a timer action/); }); it('should run action in time', async () => { - const timer = new Timer({ - description: 'running test', - timeout: 1000, - }); - - expect(await timer.run(() => 5)).toBe(5); + const timer = new Timer().schedule(1000); + await expect(timer.run('running test', () => 5)).resolves.toBe(5); }); it('should throw if an action takes longer', async () => { - const timer = new Timer({ - description: 'running this test', - timeout: 999, - }); + const timer = new Timer().schedule(999); jest.advanceTimersByTime(1000); - await expect(timer.run(() => {})) + await expect(timer.run('running this test', () => {})) .rejects.toThrowError(/Exceeded timeout of 999ms while running this test/); }); it('should throw if a sequence of actions takes longer', async () => { - const timer = new Timer({ - description: 'running this test', - timeout: 999, - }); + const timer = new Timer().schedule(999); for (let i = 0; i < 10; i++) { - await timer.run(() => {}); + await timer.run('running this test', () => {}); jest.advanceTimersByTime(100); } - await expect(timer.run(() => {})) + await expect(timer.run('running this test', () => {})) .rejects.toThrowError(/Exceeded timeout of 999ms while running this test/); }); - it('should be disposable', async () => { - const timer = new Timer({ - description: 'running this test', - timeout: 999, - }); - - timer.dispose(); - jest.advanceTimersByTime(2000); - await expect(timer.run(() => 5)).resolves.toBe(5); + it('should be clearable', async () => { + const deferred = new Deferred(); + const timer = new Timer().schedule(999); + const promise = timer.run('testing', () => deferred.promise); + setTimeout(() => deferred.resolve(5), 1500); + jest.advanceTimersByTime(100); + timer.clear(); + jest.advanceTimersByTime(1400); + await expect(promise).resolves.toBe(5); }); it('should reset timer while it is running', async () => { - const timer = new Timer({ - description: 'running this test', - timeout: 500, - }); + const timer = new Timer().schedule(500); jest.advanceTimersByTime(499); - timer.reset(100); + timer.extend(100); - jest.advanceTimersByTime(99); + jest.advanceTimersByTime(100); - await expect(timer.run(() => 5)).resolves.toBe(5); + await expect(timer.run('testing', () => 5)).resolves.toBe(5); jest.advanceTimersByTime(1); - await expect(timer.run(() => 5)) - .rejects.toThrowError(/Exceeded timeout of 100ms while running this test/); + await expect(timer.run('testing', () => 5)) + .rejects.toThrowError(/Exceeded timeout of 600ms while testing/); }); it('should reset timer after timeout', async () => { - const timer = new Timer({ - description: 'running this test', - timeout: 500, - }); + const timer = new Timer().schedule(500); jest.advanceTimersByTime(500); - await expect(timer.run(() => 5)) - .rejects.toThrowError(/Exceeded timeout of 500ms while running this test/); + await expect(timer.run('testing', () => 5)) + .rejects.toThrowError(/Exceeded timeout of 500ms while testing/); - timer.reset(100); - await expect(timer.run(() => 5)).resolves.toBe(5); + timer.extend(100); + await expect(timer.run('testing', () => 5)).resolves.toBe(5); jest.advanceTimersByTime(100); - await expect(timer.run(() => 5)) - .rejects.toThrowError(/Exceeded timeout of 100ms while running this test/); + await expect(timer.run('testing', () => 5)) + .rejects.toThrowError(/Exceeded timeout of 100ms while testing/); }); }); diff --git a/detox/src/utils/errorUtils.js b/detox/src/utils/errorUtils.js index e035038a99..f96319842b 100644 --- a/detox/src/utils/errorUtils.js +++ b/detox/src/utils/errorUtils.js @@ -1,4 +1,6 @@ const { isError } = require('lodash'); +const { deserializeError, serializeError } = require('serialize-error'); + const CLEAN_AT = /\n\s*at [\s\S]*/m; function filterErrorStack(error, predicate) { @@ -38,9 +40,27 @@ function asError(error) { return isError(error) ? error : new Error(error); } +function serializeObjectWithError(obj, errorKey) { + if (obj[errorKey]) { + return { ...obj, [errorKey]: serializeError(obj[errorKey]) }; + } + + return obj; +} + +function deserializeObjectWithError(obj, errorKey) { + if (typeof obj[errorKey] === 'object' && !(obj[errorKey] instanceof Error)) { + return { ...obj, [errorKey]: deserializeError(obj[errorKey]) }; + } + + return obj; +} + module.exports = { asError, replaceErrorStack, filterErrorStack, createErrorWithUserStack, + serializeObjectWithError, + deserializeObjectWithError, }; diff --git a/detox/test/e2e-unhappy/detox-init-timeout/testEnvironment.js b/detox/test/e2e-unhappy/detox-init-timeout/testEnvironment.js index 04ee26ffe9..c429a3fd4d 100644 --- a/detox/test/e2e-unhappy/detox-init-timeout/testEnvironment.js +++ b/detox/test/e2e-unhappy/detox-init-timeout/testEnvironment.js @@ -6,7 +6,7 @@ class CustomDetoxEnvironment extends DetoxCircusEnvironment { constructor(config, context) { super(config, context); - this.initTimeout = 30000; + this.setupTimeout = 30000; } /** @override */ diff --git a/detox/test/e2e/utils/timeoutUtils.js b/detox/test/e2e/utils/timeoutUtils.js deleted file mode 100644 index 33b32a893b..0000000000 --- a/detox/test/e2e/utils/timeoutUtils.js +++ /dev/null @@ -1,6 +0,0 @@ -const isInTimeoutTest = process.env.TIMEOUT_E2E_TEST === '1'; - -module.exports = { - initTimeout: isInTimeoutTest ? 30000 : 300000, - testTimeout: 120000, -}; \ No newline at end of file diff --git a/detox/test/types/detox-internals-tests.ts b/detox/test/types/detox-internals-tests.ts index 8d76e0ef2d..3a40801a5e 100644 --- a/detox/test/types/detox-internals-tests.ts +++ b/detox/test/types/detox-internals-tests.ts @@ -21,6 +21,7 @@ import { onTestFnStart, onTestFnSuccess, onTestStart, + reportTestResults, resolveConfig, session, tracing, @@ -194,6 +195,23 @@ async function lifecycleTest() { await onTestStart({ }); + + await reportTestResults([ + { + testFilePath: 'test1', + success: true, + }, + { + testFilePath: 'test2', + success: false, + }, + { + testFilePath: 'test1', + success: false, + testExecError: new Error('Generic test suite failure'), + isPermanentFailure: true, + }, + ]); } Promise.all([ diff --git a/docs/api/internals.md b/docs/api/internals.md index a9552c2a59..fdac13207a 100644 --- a/docs/api/internals.md +++ b/docs/api/internals.md @@ -61,16 +61,17 @@ The naming you can see adheres much to Jest Circus workflow: - `onRunDescribeFinish` - `onRunFinish` -### Reporting failed tests +### Reporting test results -`reportFailedTests` reports to Detox CLI about failed tests that could +`reportTestResults` reports to Detox CLI about failed tests that could have been re-run if `--retries` is set to a non-zero. -It takes two arguments: +It takes one argument, an array of test file reports. Each report is an object with the following properties: -- `testFilePaths` – array of failed test files' paths -- `permanent` – whether the failure is permanent, and the tests should not be re-run. -- returns a promise. +- `testFilePath` (string) — global or relative path to the failed test file; +- `success` (boolean) — whether the test passed or not; +- `testExecError` (optional error) — top-level error if the entire test file failed; +- `isPermanentFailure` (optional boolean) — if the test failed, it should tell whether the failure is permanent. Permanent failure means that the test file should not be re-run. ## Properties diff --git a/docs/config/testRunner.mdx b/docs/config/testRunner.mdx index cf90a2eaab..077bf917ab 100644 --- a/docs/config/testRunner.mdx +++ b/docs/config/testRunner.mdx @@ -133,6 +133,13 @@ DETOX_CONFIGURATION="…" jest --config e2e/jest.config.js /path/to/your/test.js # … ``` +### `testRunner.bail` \[boolean] + +Default: `false`. + +When true, tells `detox test` to cancel next retrying if it gets at least one report about a [permanent test suite failure](../api/internals.md#reporting-test-results). +Has no effect, if [`testRunner.retries`] is undefined or set to zero. + ### `testRunner.forwardEnv` \[boolean] Default: `false`. @@ -181,11 +188,12 @@ module.exports = { This is an add-on section used by our Jest integration code (but not Detox core itself). In other words, if you’re implementing (or using) a custom integration with some other test runner, feel free to define a section for yourself (e.g. `testRunner.mocha`) -### `testRunner.jest.initTimeout` \[number] +### `testRunner.jest.setupTimeout` \[number] Default: `300000` (5 minutes). -In the init phase (a part of the [environment setup](https://jestjs.io/docs/configuration/#testenvironment-string)), Detox boots the device and installs the apps. If that takes longer than the specified value, the entire test suite will be considered as failed, e.g.: +As a part of the [environment setup](https://jestjs.io/docs/configuration/#testenvironment-string)), Detox boots the device and installs the apps. +If that takes longer than the specified value, the entire test suite will be considered as failed, e.g.: ```plain text FAIL e2e/starter.test.js @@ -194,6 +202,12 @@ In the init phase (a part of the [environment setup](https://jestjs.io/docs/conf Exceeded timeout of 300000ms while setting up Detox environment ``` +### `testRunner.jest.teardownTimeout` \[number] + +Default: `30000` (30 seconds). + +If the [environment teardown](https://jestjs.io/docs/configuration/#testenvironment-string)) takes longer than the specified value, Detox will throw a timeout error. + ### `testRunner.jest.reportSpecs` \[boolean | undefined] Default: `undefined` (auto). @@ -407,3 +421,4 @@ detox test -c ios.sim.debug -- --help [`testRunner.args`]: #testrunnerargs-object [`testRunner.args.$0`]: #testrunnerargs0-string [`testRunner.inspectBrk`]: #testrunnerinspectbrk-function +[`testRunner.retries`] #testrunnerretries-number diff --git a/docs/guide/migration.md b/docs/guide/migration.md index 0583ecda0f..c5365c4de2 100644 --- a/docs/guide/migration.md +++ b/docs/guide/migration.md @@ -83,7 +83,7 @@ module.exports = CustomDetoxEnvironment; If you want, for example: -- to reduce `initTimeout` to 120 seconds, +- to reduce the init timeout to 120 seconds, - remove `SpecReporter` output, - remove `WorkerAssignReporter` output, @@ -100,7 +100,7 @@ module.exports = { }, // highlight-start jest: { - initTimeout: 120000, + setupTimeout: 120000, reportSpecs: false, reportWorkerAssign: false, }, @@ -129,7 +129,7 @@ module.exports = CustomDetoxEnvironment; Pay attention that the import has been changed to `detox/runners/jest` (previously it was `detox/runners/jest-circus`), and the reporters (`SpecReporter`, `WorkerAssignReporter`) are no longer exported. You can continue using -`initTimeout` if you want it there, but you can also delegate that to Detox config like shown earlier. +`this.initTimeout` if you rename it there to `this.setupTimeout`, but you can also delegate that to Detox config like shown earlier. ### New Jest config diff --git a/docs/guide/proguard-configuration.md b/docs/guide/proguard-configuration.md index ea3ec14240..0e53df6f6e 100644 --- a/docs/guide/proguard-configuration.md +++ b/docs/guide/proguard-configuration.md @@ -93,4 +93,5 @@ If your app already contains flavors – that makes things a bit trickier, but t A special thanks to [@GEllickson-Hover](https://github.com/GEllickson-Hover) for reporting issues related to obfuscation in [#2431](https://github.com/wix/Detox/issues/2431). [Android Reflection API]: https://developer.android.com/reference/java/lang/reflect/package-summary + [ProGuard minification]: https://developer.android.com/studio/build/shrink-code