fix(jest-test-sequencer): make --onlyFailures flag working for tests failed due to compilation errors (#15851)

Author: hainenber

CHANGELOG.md

@@ -7,6 +7,7 @@
 ### Fixes
 
 - `[jest-runtime]` Fix issue where user cannot utilize dynamic import despite specifying `--experimental-vm-modules` Node option ([#15842](https://github.com/jestjs/jest/pull/15842))
+- `[jest-test-sequencer]` Fix issue where failed tests due to compilation errors not getting re-executed even with `--onlyFailures` CLI option ([#15851](https://github.com/jestjs/jest/pull/15851))
 
 ### Chore & Maintenance
 

e2e/__tests__/onlyFailuresNonWatch.test.ts

@@ -16,46 +16,72 @@ const DIR = path.resolve(tmpdir(), 'non-watch-mode-onlyFailures');
 beforeEach(() => cleanup(DIR));
 afterEach(() => cleanup(DIR));
 
-test('onlyFailures flag works in non-watch mode', () => {
-  writeFiles(DIR, {
-    '__tests__/a.js': `
+const failedTestContents = [
+  {
+    content: {
+      '__tests__/a.js': `
     test('bar', () => { expect('bar').toBe('foo'); });
     `,
-    '__tests__/b.js': `
+      '__tests__/b.js': `
     test('foo', () => { expect('foo').toBe('foo'); });
     `,
-    'package.json': JSON.stringify({
-      jest: {
-        testEnvironment: 'node',
-      },
-    }),
-  });
-
-  let stdout, stderr;
-
-  ({stdout, stderr} = runJest(DIR));
-  expect(stdout).toBe('');
-  expect(stderr).toMatch('FAIL __tests__/a.js');
-  expect(stderr).toMatch('PASS __tests__/b.js');
-
-  // only the failed test should run and it should fail
-  ({stdout, stderr} = runJest(DIR, ['--onlyFailures']));
-  expect(stdout).toBe('');
-  expect(stderr).toMatch('FAIL __tests__/a.js');
-  expect(stderr).not.toMatch('__tests__/b.js');
-
-  // fix the failing test
-  const data = "test('bar 1', () => { expect('bar').toBe('bar'); })";
-  fs.writeFileSync(path.join(DIR, '__tests__/a.js'), data);
-
-  // only the failed test should run and it should pass
-  ({stdout, stderr} = runJest(DIR, ['--onlyFailures']));
-  expect(stdout).toBe('');
-  expect(stderr).toMatch('PASS __tests__/a.js');
-  expect(stderr).not.toMatch('__tests__/b.js');
-
-  // No test should run
-  ({stdout, stderr} = runJest(DIR, ['--onlyFailures']));
-  expect(stdout).toBe('No failed test found.');
-  expect(stderr).toBe('');
-});
+      'package.json': JSON.stringify({
+        jest: {
+          testEnvironment: 'node',
+        },
+      }),
+    },
+    name: 'failed test logic from bar != foo',
+  },
+  {
+    content: {
+      '__tests__/a.js': `
+    tes('bar', () => { expect('bar').toBe('foo'); });
+    `,
+      '__tests__/b.js': `
+    test('foo', () => { expect('foo').toBe('foo'); });
+    `,
+      'package.json': JSON.stringify({
+        jest: {
+          testEnvironment: 'node',
+        },
+      }),
+    },
+    name: 'failed test compilation from SyntaxError',
+  },
+];
+
+test.each(failedTestContents)(
+  'onlyFailures flag works in non-watch mode due to $name',
+  ({content}) => {
+    writeFiles(DIR, content);
+
+    let stdout, stderr;
+
+    ({stdout, stderr} = runJest(DIR));
+    expect(stdout).toBe('');
+    expect(stderr).toMatch('FAIL __tests__/a.js');
+    expect(stderr).toMatch('PASS __tests__/b.js');
+
+    // only the failed test should run and it should fail
+    ({stdout, stderr} = runJest(DIR, ['--onlyFailures']));
+    expect(stdout).toBe('');
+    expect(stderr).toMatch('FAIL __tests__/a.js');
+    expect(stderr).not.toMatch('__tests__/b.js');
+
+    // fix the failing test
+    const data = "test('bar 1', () => { expect('bar').toBe('bar'); })";
+    fs.writeFileSync(path.join(DIR, '__tests__/a.js'), data);
+
+    // only the failed test should run and it should pass
+    ({stdout, stderr} = runJest(DIR, ['--onlyFailures']));
+    expect(stdout).toBe('');
+    expect(stderr).toMatch('PASS __tests__/a.js');
+    expect(stderr).not.toMatch('__tests__/b.js');
+
+    // No test should run
+    ({stdout, stderr} = runJest(DIR, ['--onlyFailures']));
+    expect(stdout).toBe('No failed test found.');
+    expect(stderr).toBe('');
+  },
+);

packages/jest-test-sequencer/src/__tests__/test_sequencer.test.ts

@@ -7,7 +7,12 @@
 
 import * as path from 'path';
 import * as mockedFs from 'graceful-fs';
-import type {AggregatedResult, Test, TestContext} from '@jest/test-result';
+import type {
+  AggregatedResult,
+  SerializableError,
+  Test,
+  TestContext,
+} from '@jest/test-result';
 import {makeGlobalConfig, makeProjectConfig} from '@jest/test-utils';
 import TestSequencer from '../index';
 
@@ -137,7 +142,7 @@ test('writes the cache based on results without existing cache', async () => {
     throw new Error('File does not exist.');
   });
 
-  const testPaths = ['/test-a.js', '/test-b.js', '/test-c.js'];
+  const testPaths = ['/test-a.js', '/test-b.js', '/test-c.js', '/test-d.js'];
   const tests = await sequencer.sort(toTests(testPaths));
   sequencer.cacheResults(tests, {
     testResults: [
@@ -163,6 +168,12 @@ test('writes the cache based on results without existing cache', async () => {
         perfStats: {end: 2, runtime: 1, start: 1},
         testFilePath: '/test-x.js',
       },
+      {
+        numFailingTests: 0,
+        perfStats: {end: 2, runtime: 1, start: 1},
+        testExecError: {message: 'SyntaxError'} as SerializableError,
+        testFilePath: '/test-d.js',
+      },
     ],
   });
   const fileData = JSON.parse(
@@ -171,6 +182,7 @@ test('writes the cache based on results without existing cache', async () => {
   expect(fileData).toEqual({
     '/test-a.js': [SUCCESS, 1],
     '/test-c.js': [FAIL, 3],
+    '/test-d.js': [FAIL, 1],
   });
 });
 
@@ -198,7 +210,7 @@ test('writes the cache based on the results', async () => {
     }),
   );
 
-  const testPaths = ['/test-a.js', '/test-b.js', '/test-c.js'];
+  const testPaths = ['/test-a.js', '/test-b.js', '/test-c.js', '/test-d.js'];
   const tests = await sequencer.sort(toTests(testPaths));
   sequencer.cacheResults(tests, {
     testResults: [
@@ -223,6 +235,12 @@ test('writes the cache based on the results', async () => {
         perfStats: {end: 2, runtime: 1, start: 1},
         testFilePath: '/test-x.js',
       },
+      {
+        numFailingTests: 0,
+        perfStats: {end: 2, runtime: 1, start: 1},
+        testExecError: {message: 'SyntaxError'} as SerializableError,
+        testFilePath: '/test-d.js',
+      },
     ],
   });
   const fileData = JSON.parse(
@@ -232,6 +250,7 @@ test('writes the cache based on the results', async () => {
     '/test-a.js': [SUCCESS, 1],
     '/test-b.js': [FAIL, 1],
     '/test-c.js': [FAIL, 3],
+    '/test-d.js': [FAIL, 1],
   });
 });
 

packages/jest-test-sequencer/src/index.ts

@@ -231,7 +231,9 @@ export default class TestSequencer {
         const testRuntime =
           perf.runtime ?? test.duration ?? perf.end - perf.start;
         cache[testResult.testFilePath] = [
-          testResult.numFailingTests > 0 ? FAIL : SUCCESS,
+          testResult.numFailingTests > 0 || testResult.testExecError
+            ? FAIL
+            : SUCCESS,
           testRuntime || 0,
         ];
       }