fix(JestTestRunner): run jest with --findRelatedTests (#1235)

Running jest tests with [`--findRelatedTests`](https://jestjs.io/docs/en/cli.html#findrelatedtests-spaceseparatedlistofsourcefiles) by default. You can disable it with `jest.enableFindRelatedTests: false`

packages/stryker-api/src/test_runner/RunOptions.ts

@@ -12,6 +12,8 @@ interface RunOptions {
    * It should be loaded right after the test framework but right before any tests can run.
    */
   testHooks?: string;
+
+  mutatedFileName?: string;
 }
 
 export default RunOptions;

packages/stryker-jest-runner/README.md

@@ -40,7 +40,8 @@ The stryker-jest-runner also provides a couple of configurable options using the
 {
     jest: {
         projectType: 'custom',
-        config: require('path/to/your/custom/jestConfig.js')
+        config: require('path/to/your/custom/jestConfig.js'),
+        enableFindRelatedTests: true,
     }
 }
 ```
@@ -51,6 +52,7 @@ The stryker-jest-runner also provides a couple of configurable options using the
 | | | | `react` when you are using [create-react-app](https://github.com/facebook/create-react-app) |
 | | | | `react-ts` when you are using [create-react-app-typescript](https://github.com/wmonk/create-react-app-typescript) |
 | config (optional) | A custom Jest configuration object. You could also use `require` to load it here) | undefined | |
+| enableFindRelatedTests (optional) | Whether to run jest with the `--findRelatedTests` flag. When `true`, Jest will only run tests related to the mutated file per test. (See [_--findRelatedTests_](https://jestjs.io/docs/en/cli.html#findrelatedtests-spaceseparatedlistofsourcefiles))  | true | false |
 
 **Note:** When neither of the options are specified it will use the Jest configuration in your "package.json". \
 **Note:** the `projectType` option is ignored when the `config` option is specified.

packages/stryker-jest-runner/src/JestTestRunner.ts

@@ -1,12 +1,13 @@
 import { getLogger } from 'stryker-api/logging';
-import { RunnerOptions, RunResult, TestRunner, RunStatus, TestResult, TestStatus } from 'stryker-api/test_runner';
+import { RunnerOptions, RunResult, TestRunner, RunStatus, TestResult, TestStatus, RunOptions } from 'stryker-api/test_runner';
 import * as jest from 'jest';
 import JestTestAdapterFactory from './jestTestAdapters/JestTestAdapterFactory';
 
 export default class JestTestRunner implements TestRunner {
   private readonly log = getLogger(JestTestRunner.name);
   private readonly jestConfig: jest.Configuration;
   private readonly processEnvRef: NodeJS.ProcessEnv;
+  private readonly enableFindRelatedTests: boolean;
 
   public constructor(options: RunnerOptions, processEnvRef?: NodeJS.ProcessEnv) {
     // Make sure process can be mocked by tests by passing it in the constructor
@@ -15,19 +16,31 @@ export default class JestTestRunner implements TestRunner {
     // Get jest configuration from stryker options and assign it to jestConfig
     this.jestConfig = options.strykerOptions.jest.config;
 
+    // Get enableFindRelatedTests from stryker jest options or default to true
+    this.enableFindRelatedTests = options.strykerOptions.jest.enableFindRelatedTests;
+    if (this.enableFindRelatedTests === undefined) {
+      this.enableFindRelatedTests = true;
+    }
+
+    if (this.enableFindRelatedTests) {
+      this.log.debug('Running jest with --findRelatedTests flag. Set jest.enableFindRelatedTests to false to run all tests on every mutant.');
+    } else {
+      this.log.debug('Running jest without --findRelatedTests flag. Set jest.enableFindRelatedTests to true to run only relevant tests on every mutant.');
+    }
+
     // basePath will be used in future releases of Stryker as a way to define the project root
     // Default to process.cwd when basePath is not set for now, should be removed when issue is solved
     // https://github.com/stryker-mutator/stryker/issues/650
     this.jestConfig.rootDir = options.strykerOptions.basePath || process.cwd();
     this.log.debug(`Project root is ${this.jestConfig.rootDir}`);
   }
 
-  public async run(): Promise<RunResult> {
+  public async run(options: RunOptions): Promise<RunResult> {
     this.setNodeEnv();
 
     const jestTestRunner = JestTestAdapterFactory.getJestTestAdapter();
 
-    const { results } = await jestTestRunner.run(this.jestConfig, process.cwd());
+    const { results } = await jestTestRunner.run(this.jestConfig, process.cwd(), this.enableFindRelatedTests ? options.mutatedFileName : undefined);
 
     // Get the non-empty errorMessages from the jest RunResult, it's safe to cast to Array<string> here because we filter the empty error messages
     const errorMessages = results.testResults.map((testSuite: jest.TestResult) => testSuite.failureMessage).filter(errorMessage => (errorMessage)) as string[];

packages/stryker-jest-runner/src/jestTestAdapters/JestPromiseTestAdapter.ts

@@ -5,12 +5,16 @@ import { Configuration, runCLI, RunResult } from 'jest';
 export default class JestPromiseTestAdapter implements JestTestAdapter {
   private readonly log = getLogger(JestPromiseTestAdapter.name);
 
-  public run(jestConfig: Configuration, projectRoot: string): Promise<RunResult> {
+  public run(jestConfig: Configuration, projectRoot: string, fileNameUnderTest?: string): Promise<RunResult> {
     jestConfig.reporters = [];
     const config = JSON.stringify(jestConfig);
     this.log.trace(`Invoking Jest with config ${config}`);
+    if (fileNameUnderTest) {
+      this.log.trace(`Only running tests related to ${fileNameUnderTest}`);
+    }
 
     return runCLI({
+      ...(fileNameUnderTest && { _: [fileNameUnderTest], findRelatedTests: true}),
       config,
       runInBand: true,
       silent: true

packages/stryker-jest-runner/src/jestTestAdapters/JestTestAdapter.ts

@@ -1,5 +1,5 @@
 import { RunResult } from 'jest';
 
 export default interface JestTestAdapter {
-  run(config: object, projectRoot: string): Promise<RunResult>;
+  run(config: object, projectRoot: string, fileNameUnderTest?: string): Promise<RunResult>;
 }

packages/stryker-jest-runner/test/integration/StrykerJestRunnerSpec.ts

@@ -1,5 +1,5 @@
 import { Config } from 'stryker-api/config';
-import { RunnerOptions, RunStatus, TestStatus } from 'stryker-api/test_runner';
+import { RunnerOptions, RunStatus, TestStatus, RunOptions } from 'stryker-api/test_runner';
 import * as sinon from 'sinon';
 import { expect } from 'chai';
 import * as path from 'path';
@@ -21,9 +21,11 @@ describe('Integration test for Strykers Jest runner', () => {
   // Set timeout for integration tests to 10 seconds for travis
 
   let jestConfigEditor: JestConfigEditor;
-  let runOptions: RunnerOptions;
+  let runnerOptions: RunnerOptions;
   let processCwdStub: sinon.SinonStub;
 
+  const runOptions: RunOptions = { timeout: 0 };
+
   // Names of the tests in the example projects
   const testNames = [
     'Add should be able to add two numbers',
@@ -39,7 +41,7 @@ describe('Integration test for Strykers Jest runner', () => {
 
     jestConfigEditor = new JestConfigEditor();
 
-    runOptions = {
+    runnerOptions = {
       fileNames: [],
       port: 0,
       strykerOptions: new Config()
@@ -48,11 +50,11 @@ describe('Integration test for Strykers Jest runner', () => {
 
   it('should run tests on the example React + TypeScript project', async () => {
     processCwdStub.returns(getProjectRoot('reactTsProject'));
-    runOptions.strykerOptions.set({ jest: { projectType: 'react-ts' } });
-    jestConfigEditor.edit(runOptions.strykerOptions as Config);
+    runnerOptions.strykerOptions.set({ jest: { projectType: 'react-ts' } });
+    jestConfigEditor.edit(runnerOptions.strykerOptions as Config);
 
-    const jestTestRunner = new JestTestRunner(runOptions);
-    const result = await jestTestRunner.run();
+    const jestTestRunner = new JestTestRunner(runnerOptions);
+    const result = await jestTestRunner.run(runOptions);
 
     expect(result.status).to.equal(RunStatus.Complete);
     expect(result).to.have.property('tests');
@@ -67,10 +69,10 @@ describe('Integration test for Strykers Jest runner', () => {
   it('should run tests on the example custom project using package.json', async () => {
     processCwdStub.returns(getProjectRoot('exampleProject'));
 
-    jestConfigEditor.edit(runOptions.strykerOptions as Config);
-    const jestTestRunner = new JestTestRunner(runOptions);
+    jestConfigEditor.edit(runnerOptions.strykerOptions as Config);
+    const jestTestRunner = new JestTestRunner(runnerOptions);
 
-    const result = await jestTestRunner.run();
+    const result = await jestTestRunner.run(runOptions);
 
     expect(result.errorMessages, `Errors were: ${result.errorMessages}`).lengthOf(0);
     expect(result).to.have.property('tests');
@@ -89,10 +91,10 @@ describe('Integration test for Strykers Jest runner', () => {
   it('should run tests on the example custom project using jest.config.js', async () => {
     processCwdStub.returns(getProjectRoot('exampleProjectWithExplicitJestConfig'));
 
-    jestConfigEditor.edit(runOptions.strykerOptions as Config);
-    const jestTestRunner = new JestTestRunner(runOptions);
+    jestConfigEditor.edit(runnerOptions.strykerOptions as Config);
+    const jestTestRunner = new JestTestRunner(runnerOptions);
 
-    const result = await jestTestRunner.run();
+    const result = await jestTestRunner.run(runOptions);
 
     expect(result.errorMessages, `Errors were: ${result.errorMessages}`).lengthOf(0);
     expect(result).to.have.property('tests');

packages/stryker-jest-runner/test/unit/JestTestRunnerSpec.ts

@@ -4,11 +4,12 @@ import { Config } from 'stryker-api/config';
 import * as fakeResults from '../helpers/testResultProducer';
 import * as sinon from 'sinon';
 import { assert, expect } from 'chai';
-import { RunStatus, TestStatus } from 'stryker-api/test_runner';
+import { RunStatus, TestStatus, RunOptions } from 'stryker-api/test_runner';
 import currentLogMock from '../helpers/logMock';
 
 describe('JestTestRunner', () => {
   const basePath = '/path/to/project/root';
+  const runOptions: RunOptions = { timeout: 0 };
 
   let jestTestAdapterFactoryStub: sinon.SinonStub;
   let runJestStub: sinon.SinonStub;
@@ -44,21 +45,21 @@ describe('JestTestRunner', () => {
   });
 
   it('should call jestTestAdapterFactory "getJestTestAdapter" method to obtain a testRunner', async () => {
-    await jestTestRunner.run();
+    await jestTestRunner.run(runOptions);
 
     assert(jestTestAdapterFactoryStub.called);
   });
 
   it('should call the run function with the provided config and the projectRoot', async () => {
-    await jestTestRunner.run();
+    await jestTestRunner.run(runOptions);
 
     assert(runJestStub.called);
   });
 
   it('should call the jestTestRunner run method and return a correct runResult', async () => {
     runJestStub.resolves({ results: fakeResults.createSuccessResult() });
 
-    const result = await jestTestRunner.run();
+    const result = await jestTestRunner.run(runOptions);
 
     expect(result).to.deep.equal({
       errorMessages: [],
@@ -77,7 +78,7 @@ describe('JestTestRunner', () => {
   it('should call the jestTestRunner run method and return a skipped runResult', async () => {
     runJestStub.resolves({ results: fakeResults.createPendingResult() });
 
-    const result = await jestTestRunner.run();
+    const result = await jestTestRunner.run(runOptions);
 
     expect(result).to.deep.equal({
       errorMessages: [],
@@ -96,7 +97,7 @@ describe('JestTestRunner', () => {
   it('should call the jestTestRunner run method and return a negative runResult', async () => {
     runJestStub.resolves({ results: fakeResults.createFailResult() });
 
-    const result = await jestTestRunner.run();
+    const result = await jestTestRunner.run(runOptions);
 
     expect(result).to.deep.equal({
       errorMessages: ['test failed - App.test.js'],
@@ -131,7 +132,7 @@ describe('JestTestRunner', () => {
   it('should return an error result when a runtime error occurs', async () => {
     runJestStub.resolves({ results: { testResults: [], numRuntimeErrorTestSuites: 1 } });
 
-    const result = await jestTestRunner.run();
+    const result = await jestTestRunner.run(runOptions);
 
     expect(result).to.deep.equal({
       errorMessages: [],
@@ -141,15 +142,15 @@ describe('JestTestRunner', () => {
   });
 
   it('should set process.env.NODE_ENV to \'test\' when process.env.NODE_ENV is null', async () => {
-    await jestTestRunner.run();
+    await jestTestRunner.run(runOptions);
 
     expect(processEnvMock.NODE_ENV).to.equal('test');
   });
 
   it('should keep the value set in process.env.NODE_ENV if not null', async () => {
     processEnvMock.NODE_ENV = 'stryker';
 
-    await jestTestRunner.run();
+    await jestTestRunner.run(runOptions);
 
     expect(processEnvMock.NODE_ENV).to.equal('stryker');
   });

packages/stryker-jest-runner/test/unit/jestTestAdapters/JestPromiseTestAdapterSpec.ts

@@ -9,6 +9,7 @@ describe('JestPromiseTestAdapter', () => {
   let runCLIStub: sinon.SinonStub;
 
   const projectRoot = '/path/to/project';
+  const fileNameUnderTest = '/path/to/file';
   const jestConfig: any = { rootDir: projectRoot };
 
   beforeEach(() => {
@@ -37,6 +38,18 @@ describe('JestPromiseTestAdapter', () => {
     }, [projectRoot]));
   });
 
+  it('should call the runCLI method with the --findRelatedTests flag', async () => {
+    await jestPromiseTestAdapter.run(jestConfig, projectRoot, fileNameUnderTest);
+
+    assert(runCLIStub.calledWith({
+      _: [fileNameUnderTest],
+      config: JSON.stringify({ rootDir: projectRoot, reporters: [] }),
+      findRelatedTests: true,
+      runInBand: true,
+      silent: true
+    }, [projectRoot]));
+  });
+
   it('should call the runCLI method and return the test result', async () => {
     const result = await jestPromiseTestAdapter.run(jestConfig, projectRoot);
 
@@ -50,6 +63,21 @@ describe('JestPromiseTestAdapter', () => {
     });
   });
 
+  it('should call the runCLI method and return the test result when run with --findRelatedTests flag', async () => {
+    const result = await jestPromiseTestAdapter.run(jestConfig, projectRoot, fileNameUnderTest);
+
+    expect(result).to.deep.equal({
+      config: {
+        _: [fileNameUnderTest],
+        config: JSON.stringify({ rootDir: projectRoot, reporters: [] }),
+        findRelatedTests: true,
+        runInBand: true,
+        silent: true
+      },
+      result: 'testResult'
+    });
+  });
+
   it('should trace log a message when jest is invoked', async () => {
     await jestPromiseTestAdapter.run(jestConfig, projectRoot);
 

packages/stryker-jest-runner/types/jest/index.d.ts

@@ -8,6 +8,8 @@ declare namespace Jest {
     config: string;
     runInBand: boolean;
     silent: boolean;
+    findRelatedTests?: boolean;
+    _?: string[];
   }
 
   // Taken from https://goo.gl/qHifyP, removed all stuff that we are not using
@@ -18,7 +20,7 @@ declare namespace Jest {
     collectCoverage: boolean;
     verbose: boolean;
     testResultsProcessor: Maybe<string>;
-    testEnvironment: string
+    testEnvironment: string;
   }
 
   interface RunResult {

packages/stryker/src/Sandbox.ts

@@ -44,8 +44,8 @@ export default class Sandbox {
     return sandbox.initialize().then(() => sandbox);
   }
 
-  public run(timeout: number, testHooks: string | undefined): Promise<RunResult> {
-    return this.testRunner.run({ timeout, testHooks });
+  public run(timeout: number, testHooks: string | undefined, mutatedFileName?: string): Promise<RunResult> {
+    return this.testRunner.run({ timeout, testHooks, mutatedFileName });
   }
 
   public dispose(): Promise<void> {
@@ -58,7 +58,7 @@ export default class Sandbox {
       this.log.warn(`Failed find coverage data for this mutant, running all tests. This might have an impact on performance: ${transpiledMutant.mutant.toString()}`);
     }
     await Promise.all(mutantFiles.map(mutatedFile => this.writeFileInSandbox(mutatedFile)));
-    const runResult = await this.run(this.calculateTimeout(transpiledMutant.mutant), this.getFilterTestsHooks(transpiledMutant.mutant));
+    const runResult = await this.run(this.calculateTimeout(transpiledMutant.mutant), this.getFilterTestsHooks(transpiledMutant.mutant), this.fileMap[transpiledMutant.mutant.fileName]);
     await this.reset(mutantFiles);
     return runResult;
   }

packages/stryker/test/unit/SandboxSpec.ts

@@ -144,8 +144,19 @@ describe('Sandbox', () => {
       const sut = await Sandbox.create(options, SANDBOX_INDEX, files, null, 0, LOGGING_CONTEXT);
       await sut.run(231313, 'hooks');
       expect(testRunner.run).to.have.been.calledWith({
+        mutatedFileName: undefined,
         testHooks: 'hooks',
-        timeout: 231313
+        timeout: 231313,
+      });
+    });
+
+    it('should run the testRunner with mutatedFileName', async () => {
+      const sut = await Sandbox.create(options, SANDBOX_INDEX, files, null, 0, LOGGING_CONTEXT);
+      await sut.run(231313, 'hooks', 'path/to/file');
+      expect(testRunner.run).to.have.been.calledWith({
+        mutatedFileName: 'path/to/file',
+        testHooks: 'hooks',
+        timeout: 231313,
       });
     });
   });
@@ -196,13 +207,17 @@ describe('Sandbox', () => {
       expect(testFrameworkStub.filter).to.have.been.calledWith(transpiledMutant.mutant.selectedTests);
     });
 
-    it('should provide the filter code as testHooks and correct timeout', async () => {
+    it('should provide the filter code as testHooks, correct timeout and mutatedFileName', async () => {
       options.timeoutMS = 1000;
       const overheadTimeMS = 42;
       const totalTimeSpend = 12;
       const sut = await Sandbox.create(options, SANDBOX_INDEX, files, testFrameworkStub, overheadTimeMS, LOGGING_CONTEXT);
       await sut.runMutant(transpiledMutant);
-      const expectedRunOptions = { testHooks: wrapInClosure(testFilterCodeFragment), timeout: totalTimeSpend * options.timeoutFactor + options.timeoutMS + overheadTimeMS };
+      const expectedRunOptions = {
+        mutatedFileName: path.resolve('random-folder-3', 'file1'),
+        testHooks: wrapInClosure(testFilterCodeFragment),
+        timeout: totalTimeSpend * options.timeoutFactor + options.timeoutMS + overheadTimeMS
+      };
       expect(testRunner.run).calledWith(expectedRunOptions);
     });