feat(jest): new listener events (#3626)

detox/index.d.ts

@@ -139,9 +139,13 @@ declare global {
  */
  jest?: {
  /**
- * Device init timeout
+ * Environment setup timeout
  */
- initTimeout?: number | undefined;
+ setupTimeout?: number | undefined;
+ /**
+ * Environment teardown timeout
+ */
+ teardownTimeout?: number | undefined;
  /**
  * Insist on CLI-based retry mechanism even when the failed tests have been handled
  * by jest.retryTimes(n) mechanism from Jest Circus.
@@ -154,6 +158,16 @@ declare global {
  * Retries count. Zero means a single attempt to run tests.
  */
  retries?: number;
+ /**
+ * When true, tells Detox CLI to cancel next retrying if it gets
+ * at least one report about a permanent test suite failure.
+ * Has no effect, if {@link DetoxTestRunnerConfig#retries} is
+ * undefined or set to zero.
+ *
+ * @default false
+ * @see {DetoxInternals.DetoxTestFileReport#isPermanentFailure}
+ */
+ bail?: boolean;
  /**
  * Custom handler to process --inspect-brk CLI flag.
  * Use it when you rely on another test runner than Jest.

detox/internals.d.ts

@@ -54,14 +54,13 @@ declare global {
  onRunFinish(event: unknown): Promise<void>;
 
  /**
- * Reports to Detox CLI about failed tests that could have been re-run if
+ * Reports to Detox CLI about passed and failed test files.
+ * The failed test files might be re-run again if
  * {@link Detox.DetoxTestRunnerConfig#retries} is set to a non-zero.
  *
- * @param testFilePaths array of failed test files' paths
- * @param permanent whether the failure is permanent, and the tests
- * should not be re-run.
+ * @param testResults - reports about test files
  */
- reportFailedTests(testFilePaths: string[], permanent?: boolean): Promise<void>;
+ reportTestResults(testResults: DetoxTestFileReport[]): Promise<void>;
  // endregion
 
  readonly config: RuntimeConfig;
@@ -92,7 +91,7 @@ declare global {
  testRunnerArgv: Record<string, unknown>;
  override: Partial<Detox.DetoxConfig>;
  /** @inheritDoc */
- global: NodeJS.Global;
+ global: NodeJS.Global | {};
  /**
  * Worker ID. Used to distinguish allocated workers in parallel test execution environment.
  *
@@ -111,7 +110,7 @@ declare global {
  * {@link DetoxInternals.Facade#setup} might override {@link Console} methods
  * to integrate it with Detox loggeing subsystem.
  */
- global: NodeJS.Global;
+ global: NodeJS.Global | {};
  /**
  * Worker ID. Used to distinguish allocated workers in parallel test execution environment.
  *
@@ -120,25 +119,48 @@ declare global {
  workerId: string;
  };
 
+ type DetoxTestFileReport = {
+ /**
+ * Global or relative path to the failed test file.
+ */
+ testFilePath: string;
+ /**
+ * Whether the test passed or not.
+ */
+ success: boolean;
+ /**
+ * Top-level error if the entire test file failed.
+ */
+ testExecError?: { name?: string; message: string; stack?: string; };
+ /**
+ * If the test failed, it should tell whether the failure is permanent.
+ * Permanent failure means that the test file should not be re-run.
+ *
+ * @default false
+ * @see {Detox.DetoxTestRunnerConfig#retries}
+ */
+ isPermanentFailure?: boolean;
+ };
+
  type SessionState = Readonly<{
  /**
  * Randomly generated ID for the entire Detox test session, including retries.
  */
  id: string;
  /**
- * Permanently failed test file paths.
+ * Results of test file executions. Primarily used for Detox CLI retry mechanism.
  */
- failedTestFiles: string[];
- /**
- * Failed test file paths suggested to retry via Detox CLI mechanism.
- */
- testFilesToRetry: string[];
+ testResults: DetoxTestFileReport[];
  /**
  * Retry index of the test session: 0..retriesCount.
  */
  testSessionIndex: number;
  /**
- * TODO
+ * Count of Detox contexts with a worker installed.
+ * Oversimplified, it reflects the count of allocated devices in the current test session.
+ *
+ * @see {Facade#init}
+ * @see {Facade#installWorker}
  */
  workersCount: number;
  }>;

detox/local-cli/init.js

@@ -76,7 +76,7 @@ function createDefaultConfigurations() {
  config: 'e2e/jest.config.js',
  },
  jest: {
- initTimeout: 120000,
+ setupTimeout: 120000,
  },
  },
  apps: {

detox/local-cli/test.test.js

@@ -8,6 +8,8 @@ jest.mock('../src/devices/DeviceRegistry');
 jest.mock('../src/devices/allocation/drivers/android/genycloud/GenyDeviceRegistryFactory');
 jest.mock('./utils/jestInternals');
 
+const cp = require('child_process');
+const cpSpawn = cp.spawn;
 const os = require('os');
 const path = require('path');
 
@@ -28,6 +30,10 @@ describe('CLI', () => {
  let GenyDeviceRegistryFactory;
  let jestInternals;
 
+ afterEach(() => {
+ cp.spawn = cpSpawn;
+ });
+
  beforeEach(() => {
  _cliCallDump = undefined;
  _env = process.env;
@@ -139,11 +145,28 @@ describe('CLI', () => {
  });
 
  test.each([['-R'], ['--retries']])('%s <value> should execute unsuccessful run N extra times', async (__retries) => {
+ function toTestResult(testFilePath) {
+ return {
+ testFilePath,
+ success: false,
+ isPermanentFailure: false,
+ };
+ }
+
  const context = require('../internals');
- context.session.testFilesToRetry = ['e2e/failing1.test.js', 'e2e/failing2.test.js'];
- context.session.testFilesToRetry.splice = jest.fn()
- .mockImplementationOnce(() => ['e2e/failing1.test.js', 'e2e/failing2.test.js'])
- .mockImplementationOnce(() => ['e2e/failing2.test.js']);
+
+ jest.spyOn(cp, 'spawn')
+ .mockImplementationOnce((...args) => {
+ context.session.testResults = ['e2e/failing1.test.js', 'e2e/failing2.test.js'].map(toTestResult);
+ return cpSpawn(...args);
+ })
+ .mockImplementationOnce((...args) => {
+ context.session.testResults = ['e2e/failing2.test.js'].map(toTestResult);
+ return cpSpawn(...args);
+ })
+ .mockImplementationOnce((...args) => {
+ return cpSpawn(...args);
+ });
 
  mockExitCode(1);
 
@@ -154,18 +177,35 @@ describe('CLI', () => {
  expect(cliCall(2).argv).toEqual([expect.stringMatching(/executable$/), '--config', 'e2e/config.json', 'e2e/failing2.test.js']);
  });
 
- test.each([['-R'], ['--retries']])('%s <value> should bail if has permanently failed tests', async (__retries) => {
- const context = require('../internals');
- context.session.failedTestFiles = ['e2e/failing1.test.js'];
- context.session.testFilesToRetry = ['e2e/failing2.test.js'];
+ describe('when there are permanently failed tests', () => {
+ beforeEach(() => {
+ const context = require('../internals');
+ context.session.testResults = ['e2e/failing1.test.js', 'e2e/failing2.test.js'].map((testFilePath, index) => ({
+ testFilePath,
+ success: false,
+ isPermanentFailure: index > 0,
+ }));
 
- mockExitCode(1);
+ mockExitCode(1);
+ });
 
- await run(__retries, 2).catch(_.noop);
+ test.each([['-R'], ['--retries']])('%s <value> should not bail by default', async (__retries) => {
+ await run(__retries, 2).catch(_.noop);
 
- expect(cliCall(0).env).not.toHaveProperty('DETOX_RERUN_INDEX');
- expect(cliCall(0).argv).toEqual([expect.stringMatching(/executable$/), '--config', 'e2e/config.json']);
- expect(cliCall(1)).toBe(null);
+ expect(cliCall(0).env).not.toHaveProperty('DETOX_RERUN_INDEX');
+ expect(cliCall(0).argv).toEqual([expect.stringMatching(/executable$/), '--config', 'e2e/config.json']);
+ expect(cliCall(1).argv).toEqual([expect.stringMatching(/executable$/), '--config', 'e2e/config.json', 'e2e/failing1.test.js']);
+ // note that it does not take the permanently failed test
+ });
+
+ test.each([['-R'], ['--retries']])('%s <value> should bail if configured', async (__retries) => {
+ detoxConfig.testRunner.bail = true;
+ await run(__retries, 2).catch(_.noop);
+
+ expect(cliCall(0).env).not.toHaveProperty('DETOX_RERUN_INDEX');
+ expect(cliCall(0).argv).toEqual([expect.stringMatching(/executable$/), '--config', 'e2e/config.json']);
+ expect(cliCall(1)).toBe(null);
+ });
  });
 
  test.each([['-R'], ['--retries']])('%s <value> should not restart test runner if there are no failing tests paths', async (__retries) => {
@@ -178,7 +218,11 @@ describe('CLI', () => {
 
  test.each([['-R'], ['--retries']])('%s <value> should retain -- <...explicitPassthroughArgs>', async (__retries) => {
  const context = require('../internals');
- context.session.testFilesToRetry = ['tests/failing.test.js'];
+ context.session.testResults = [{
+ testFilePath: 'tests/failing.test.js',
+ success: false,
+ isPermanentFailure: false,
+ }];
 
  mockExitCode(1);
 

detox/local-cli/testCommand/TestRunnerCommand.js

@@ -99,13 +99,20 @@ class TestRunnerCommand {
  } catch (e) {
  launchError = e;
 
- const { failedTestFiles, testFilesToRetry } = detox.session;
- if (!_.isEmpty(failedTestFiles) || _.isEmpty(testFilesToRetry)) {
+ const failedTestFiles = detox.session.testResults.filter(r => !r.success);
+
+ const { bail } = detox.config.testRunner;
+ if (bail && failedTestFiles.some(r => r.isPermanentFailure)) {
+ throw e;
+ }
+
+ const testFilesToRetry = failedTestFiles.filter(r => !r.isPermanentFailure).map(r => r.testFilePath);
+ if (_.isEmpty(testFilesToRetry)) {
  throw e;
  }
 
  if (--runsLeft > 0) {
- this._argv._ = testFilesToRetry.splice(0, Infinity);
+ this._argv._ = testFilesToRetry;
  // @ts-ignore
  detox.session.testSessionIndex++; // it is always a primary context, so we can update it
  }

detox/runners/jest/reporters/DetoxReporter.js

@@ -1,5 +1,25 @@
-const { VerboseReporter: JestVerboseReporter } = require('@jest/reporters'); // eslint-disable-line node/no-extraneous-require
+const resolveFrom = require('resolve-from');
+/** @type {typeof import('@jest/reporters').VerboseReporter} */
+const JestVerboseReporter = require(resolveFrom(process.cwd(), '@jest/reporters')).VerboseReporter;
 
-class DetoxReporter extends JestVerboseReporter {}
+const { config, reportTestResults } = require('../../../internals');
+
+class DetoxReporter extends JestVerboseReporter {
+ /**
+ * @param {import('@jest/test-result').TestResult} testResult
+ */
+ async onTestResult(test, testResult, results) {
+ await super.onTestResult(test, testResult, results);
+
+ await reportTestResults([{
+ success: !testResult.failureMessage,
+ testFilePath: testResult.testFilePath,
+ testExecError: testResult.testExecError,
+ isPermanentFailure: config.testRunner.jest.retryAfterCircusRetries
+ ? false
+ : testResult.testResults.some(r => r.invocations > 1)
+ }]);
+ }
+}
 
 module.exports = DetoxReporter;