From 5e0790e039ecd16db929b4d06cc888f5c93a2ba7 Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 29 Nov 2018 13:04:46 +0100 Subject: [PATCH] fix(JestTestRunner): run jest with --findRelatedTests (#1235) Running jest tests with [`--findRelatedTests`](https://jestjs.io/docs/en/cli.html#findrelatedtests-spaceseparatedlistofsourcefiles) by default. You can disable it with `jest.enableFindRelatedTests: false` --- .../stryker-api/src/test_runner/RunOptions.ts | 2 ++ packages/stryker-jest-runner/README.md | 4 ++- .../stryker-jest-runner/src/JestTestRunner.ts | 19 +++++++++++-- .../JestPromiseTestAdapter.ts | 6 +++- .../src/jestTestAdapters/JestTestAdapter.ts | 2 +- .../test/integration/StrykerJestRunnerSpec.ts | 28 ++++++++++--------- .../test/unit/JestTestRunnerSpec.ts | 19 +++++++------ .../JestPromiseTestAdapterSpec.ts | 28 +++++++++++++++++++ .../stryker-jest-runner/types/jest/index.d.ts | 4 ++- packages/stryker/src/Sandbox.ts | 6 ++-- packages/stryker/test/unit/SandboxSpec.ts | 21 ++++++++++++-- 11 files changed, 104 insertions(+), 35 deletions(-) diff --git a/packages/stryker-api/src/test_runner/RunOptions.ts b/packages/stryker-api/src/test_runner/RunOptions.ts index 1b2ba15db4..4474c12b9a 100644 --- a/packages/stryker-api/src/test_runner/RunOptions.ts +++ b/packages/stryker-api/src/test_runner/RunOptions.ts @@ -12,6 +12,8 @@ interface RunOptions { * It should be loaded right after the test framework but right before any tests can run. */ testHooks?: string; + + mutatedFileName?: string; } export default RunOptions; diff --git a/packages/stryker-jest-runner/README.md b/packages/stryker-jest-runner/README.md index 36baf4721c..85a7ccf80b 100644 --- a/packages/stryker-jest-runner/README.md +++ b/packages/stryker-jest-runner/README.md @@ -40,7 +40,8 @@ The stryker-jest-runner also provides a couple of configurable options using the { jest: { projectType: 'custom', - config: require('path/to/your/custom/jestConfig.js') + config: require('path/to/your/custom/jestConfig.js'), + enableFindRelatedTests: true, } } ``` @@ -51,6 +52,7 @@ The stryker-jest-runner also provides a couple of configurable options using the | | | | `react` when you are using [create-react-app](https://github.com/facebook/create-react-app) | | | | | `react-ts` when you are using [create-react-app-typescript](https://github.com/wmonk/create-react-app-typescript) | | config (optional) | A custom Jest configuration object. You could also use `require` to load it here) | undefined | | +| enableFindRelatedTests (optional) | Whether to run jest with the `--findRelatedTests` flag. When `true`, Jest will only run tests related to the mutated file per test. (See [_--findRelatedTests_](https://jestjs.io/docs/en/cli.html#findrelatedtests-spaceseparatedlistofsourcefiles)) | true | false | **Note:** When neither of the options are specified it will use the Jest configuration in your "package.json". \ **Note:** the `projectType` option is ignored when the `config` option is specified. diff --git a/packages/stryker-jest-runner/src/JestTestRunner.ts b/packages/stryker-jest-runner/src/JestTestRunner.ts index 2949bb77f7..3a246e4c39 100644 --- a/packages/stryker-jest-runner/src/JestTestRunner.ts +++ b/packages/stryker-jest-runner/src/JestTestRunner.ts @@ -1,5 +1,5 @@ import { getLogger } from 'stryker-api/logging'; -import { RunnerOptions, RunResult, TestRunner, RunStatus, TestResult, TestStatus } from 'stryker-api/test_runner'; +import { RunnerOptions, RunResult, TestRunner, RunStatus, TestResult, TestStatus, RunOptions } from 'stryker-api/test_runner'; import * as jest from 'jest'; import JestTestAdapterFactory from './jestTestAdapters/JestTestAdapterFactory'; @@ -7,6 +7,7 @@ export default class JestTestRunner implements TestRunner { private readonly log = getLogger(JestTestRunner.name); private readonly jestConfig: jest.Configuration; private readonly processEnvRef: NodeJS.ProcessEnv; + private readonly enableFindRelatedTests: boolean; public constructor(options: RunnerOptions, processEnvRef?: NodeJS.ProcessEnv) { // Make sure process can be mocked by tests by passing it in the constructor @@ -15,6 +16,18 @@ export default class JestTestRunner implements TestRunner { // Get jest configuration from stryker options and assign it to jestConfig this.jestConfig = options.strykerOptions.jest.config; + // Get enableFindRelatedTests from stryker jest options or default to true + this.enableFindRelatedTests = options.strykerOptions.jest.enableFindRelatedTests; + if (this.enableFindRelatedTests === undefined) { + this.enableFindRelatedTests = true; + } + + if (this.enableFindRelatedTests) { + this.log.debug('Running jest with --findRelatedTests flag. Set jest.enableFindRelatedTests to false to run all tests on every mutant.'); + } else { + this.log.debug('Running jest without --findRelatedTests flag. Set jest.enableFindRelatedTests to true to run only relevant tests on every mutant.'); + } + // basePath will be used in future releases of Stryker as a way to define the project root // Default to process.cwd when basePath is not set for now, should be removed when issue is solved // https://github.com/stryker-mutator/stryker/issues/650 @@ -22,12 +35,12 @@ export default class JestTestRunner implements TestRunner { this.log.debug(`Project root is ${this.jestConfig.rootDir}`); } - public async run(): Promise { + public async run(options: RunOptions): Promise { this.setNodeEnv(); const jestTestRunner = JestTestAdapterFactory.getJestTestAdapter(); - const { results } = await jestTestRunner.run(this.jestConfig, process.cwd()); + const { results } = await jestTestRunner.run(this.jestConfig, process.cwd(), this.enableFindRelatedTests ? options.mutatedFileName : undefined); // Get the non-empty errorMessages from the jest RunResult, it's safe to cast to Array here because we filter the empty error messages const errorMessages = results.testResults.map((testSuite: jest.TestResult) => testSuite.failureMessage).filter(errorMessage => (errorMessage)) as string[]; diff --git a/packages/stryker-jest-runner/src/jestTestAdapters/JestPromiseTestAdapter.ts b/packages/stryker-jest-runner/src/jestTestAdapters/JestPromiseTestAdapter.ts index a0d454ed76..a082b5ce5a 100644 --- a/packages/stryker-jest-runner/src/jestTestAdapters/JestPromiseTestAdapter.ts +++ b/packages/stryker-jest-runner/src/jestTestAdapters/JestPromiseTestAdapter.ts @@ -5,12 +5,16 @@ import { Configuration, runCLI, RunResult } from 'jest'; export default class JestPromiseTestAdapter implements JestTestAdapter { private readonly log = getLogger(JestPromiseTestAdapter.name); - public run(jestConfig: Configuration, projectRoot: string): Promise { + public run(jestConfig: Configuration, projectRoot: string, fileNameUnderTest?: string): Promise { jestConfig.reporters = []; const config = JSON.stringify(jestConfig); this.log.trace(`Invoking Jest with config ${config}`); + if (fileNameUnderTest) { + this.log.trace(`Only running tests related to ${fileNameUnderTest}`); + } return runCLI({ + ...(fileNameUnderTest && { _: [fileNameUnderTest], findRelatedTests: true}), config, runInBand: true, silent: true diff --git a/packages/stryker-jest-runner/src/jestTestAdapters/JestTestAdapter.ts b/packages/stryker-jest-runner/src/jestTestAdapters/JestTestAdapter.ts index c2484d30dc..b8e60d204b 100644 --- a/packages/stryker-jest-runner/src/jestTestAdapters/JestTestAdapter.ts +++ b/packages/stryker-jest-runner/src/jestTestAdapters/JestTestAdapter.ts @@ -1,5 +1,5 @@ import { RunResult } from 'jest'; export default interface JestTestAdapter { - run(config: object, projectRoot: string): Promise; + run(config: object, projectRoot: string, fileNameUnderTest?: string): Promise; } diff --git a/packages/stryker-jest-runner/test/integration/StrykerJestRunnerSpec.ts b/packages/stryker-jest-runner/test/integration/StrykerJestRunnerSpec.ts index 9090c6c18b..20aee9adac 100644 --- a/packages/stryker-jest-runner/test/integration/StrykerJestRunnerSpec.ts +++ b/packages/stryker-jest-runner/test/integration/StrykerJestRunnerSpec.ts @@ -1,5 +1,5 @@ import { Config } from 'stryker-api/config'; -import { RunnerOptions, RunStatus, TestStatus } from 'stryker-api/test_runner'; +import { RunnerOptions, RunStatus, TestStatus, RunOptions } from 'stryker-api/test_runner'; import * as sinon from 'sinon'; import { expect } from 'chai'; import * as path from 'path'; @@ -21,9 +21,11 @@ describe('Integration test for Strykers Jest runner', () => { // Set timeout for integration tests to 10 seconds for travis let jestConfigEditor: JestConfigEditor; - let runOptions: RunnerOptions; + let runnerOptions: RunnerOptions; let processCwdStub: sinon.SinonStub; + const runOptions: RunOptions = { timeout: 0 }; + // Names of the tests in the example projects const testNames = [ 'Add should be able to add two numbers', @@ -39,7 +41,7 @@ describe('Integration test for Strykers Jest runner', () => { jestConfigEditor = new JestConfigEditor(); - runOptions = { + runnerOptions = { fileNames: [], port: 0, strykerOptions: new Config() @@ -48,11 +50,11 @@ describe('Integration test for Strykers Jest runner', () => { it('should run tests on the example React + TypeScript project', async () => { processCwdStub.returns(getProjectRoot('reactTsProject')); - runOptions.strykerOptions.set({ jest: { projectType: 'react-ts' } }); - jestConfigEditor.edit(runOptions.strykerOptions as Config); + runnerOptions.strykerOptions.set({ jest: { projectType: 'react-ts' } }); + jestConfigEditor.edit(runnerOptions.strykerOptions as Config); - const jestTestRunner = new JestTestRunner(runOptions); - const result = await jestTestRunner.run(); + const jestTestRunner = new JestTestRunner(runnerOptions); + const result = await jestTestRunner.run(runOptions); expect(result.status).to.equal(RunStatus.Complete); expect(result).to.have.property('tests'); @@ -67,10 +69,10 @@ describe('Integration test for Strykers Jest runner', () => { it('should run tests on the example custom project using package.json', async () => { processCwdStub.returns(getProjectRoot('exampleProject')); - jestConfigEditor.edit(runOptions.strykerOptions as Config); - const jestTestRunner = new JestTestRunner(runOptions); + jestConfigEditor.edit(runnerOptions.strykerOptions as Config); + const jestTestRunner = new JestTestRunner(runnerOptions); - const result = await jestTestRunner.run(); + const result = await jestTestRunner.run(runOptions); expect(result.errorMessages, `Errors were: ${result.errorMessages}`).lengthOf(0); expect(result).to.have.property('tests'); @@ -89,10 +91,10 @@ describe('Integration test for Strykers Jest runner', () => { it('should run tests on the example custom project using jest.config.js', async () => { processCwdStub.returns(getProjectRoot('exampleProjectWithExplicitJestConfig')); - jestConfigEditor.edit(runOptions.strykerOptions as Config); - const jestTestRunner = new JestTestRunner(runOptions); + jestConfigEditor.edit(runnerOptions.strykerOptions as Config); + const jestTestRunner = new JestTestRunner(runnerOptions); - const result = await jestTestRunner.run(); + const result = await jestTestRunner.run(runOptions); expect(result.errorMessages, `Errors were: ${result.errorMessages}`).lengthOf(0); expect(result).to.have.property('tests'); diff --git a/packages/stryker-jest-runner/test/unit/JestTestRunnerSpec.ts b/packages/stryker-jest-runner/test/unit/JestTestRunnerSpec.ts index 2b095f6cd8..daf9588a5e 100644 --- a/packages/stryker-jest-runner/test/unit/JestTestRunnerSpec.ts +++ b/packages/stryker-jest-runner/test/unit/JestTestRunnerSpec.ts @@ -4,11 +4,12 @@ import { Config } from 'stryker-api/config'; import * as fakeResults from '../helpers/testResultProducer'; import * as sinon from 'sinon'; import { assert, expect } from 'chai'; -import { RunStatus, TestStatus } from 'stryker-api/test_runner'; +import { RunStatus, TestStatus, RunOptions } from 'stryker-api/test_runner'; import currentLogMock from '../helpers/logMock'; describe('JestTestRunner', () => { const basePath = '/path/to/project/root'; + const runOptions: RunOptions = { timeout: 0 }; let jestTestAdapterFactoryStub: sinon.SinonStub; let runJestStub: sinon.SinonStub; @@ -44,13 +45,13 @@ describe('JestTestRunner', () => { }); it('should call jestTestAdapterFactory "getJestTestAdapter" method to obtain a testRunner', async () => { - await jestTestRunner.run(); + await jestTestRunner.run(runOptions); assert(jestTestAdapterFactoryStub.called); }); it('should call the run function with the provided config and the projectRoot', async () => { - await jestTestRunner.run(); + await jestTestRunner.run(runOptions); assert(runJestStub.called); }); @@ -58,7 +59,7 @@ describe('JestTestRunner', () => { it('should call the jestTestRunner run method and return a correct runResult', async () => { runJestStub.resolves({ results: fakeResults.createSuccessResult() }); - const result = await jestTestRunner.run(); + const result = await jestTestRunner.run(runOptions); expect(result).to.deep.equal({ errorMessages: [], @@ -77,7 +78,7 @@ describe('JestTestRunner', () => { it('should call the jestTestRunner run method and return a skipped runResult', async () => { runJestStub.resolves({ results: fakeResults.createPendingResult() }); - const result = await jestTestRunner.run(); + const result = await jestTestRunner.run(runOptions); expect(result).to.deep.equal({ errorMessages: [], @@ -96,7 +97,7 @@ describe('JestTestRunner', () => { it('should call the jestTestRunner run method and return a negative runResult', async () => { runJestStub.resolves({ results: fakeResults.createFailResult() }); - const result = await jestTestRunner.run(); + const result = await jestTestRunner.run(runOptions); expect(result).to.deep.equal({ errorMessages: ['test failed - App.test.js'], @@ -131,7 +132,7 @@ describe('JestTestRunner', () => { it('should return an error result when a runtime error occurs', async () => { runJestStub.resolves({ results: { testResults: [], numRuntimeErrorTestSuites: 1 } }); - const result = await jestTestRunner.run(); + const result = await jestTestRunner.run(runOptions); expect(result).to.deep.equal({ errorMessages: [], @@ -141,7 +142,7 @@ describe('JestTestRunner', () => { }); it('should set process.env.NODE_ENV to \'test\' when process.env.NODE_ENV is null', async () => { - await jestTestRunner.run(); + await jestTestRunner.run(runOptions); expect(processEnvMock.NODE_ENV).to.equal('test'); }); @@ -149,7 +150,7 @@ describe('JestTestRunner', () => { it('should keep the value set in process.env.NODE_ENV if not null', async () => { processEnvMock.NODE_ENV = 'stryker'; - await jestTestRunner.run(); + await jestTestRunner.run(runOptions); expect(processEnvMock.NODE_ENV).to.equal('stryker'); }); diff --git a/packages/stryker-jest-runner/test/unit/jestTestAdapters/JestPromiseTestAdapterSpec.ts b/packages/stryker-jest-runner/test/unit/jestTestAdapters/JestPromiseTestAdapterSpec.ts index f2b0c99af2..af24f3b418 100644 --- a/packages/stryker-jest-runner/test/unit/jestTestAdapters/JestPromiseTestAdapterSpec.ts +++ b/packages/stryker-jest-runner/test/unit/jestTestAdapters/JestPromiseTestAdapterSpec.ts @@ -9,6 +9,7 @@ describe('JestPromiseTestAdapter', () => { let runCLIStub: sinon.SinonStub; const projectRoot = '/path/to/project'; + const fileNameUnderTest = '/path/to/file'; const jestConfig: any = { rootDir: projectRoot }; beforeEach(() => { @@ -37,6 +38,18 @@ describe('JestPromiseTestAdapter', () => { }, [projectRoot])); }); + it('should call the runCLI method with the --findRelatedTests flag', async () => { + await jestPromiseTestAdapter.run(jestConfig, projectRoot, fileNameUnderTest); + + assert(runCLIStub.calledWith({ + _: [fileNameUnderTest], + config: JSON.stringify({ rootDir: projectRoot, reporters: [] }), + findRelatedTests: true, + runInBand: true, + silent: true + }, [projectRoot])); + }); + it('should call the runCLI method and return the test result', async () => { const result = await jestPromiseTestAdapter.run(jestConfig, projectRoot); @@ -50,6 +63,21 @@ describe('JestPromiseTestAdapter', () => { }); }); + it('should call the runCLI method and return the test result when run with --findRelatedTests flag', async () => { + const result = await jestPromiseTestAdapter.run(jestConfig, projectRoot, fileNameUnderTest); + + expect(result).to.deep.equal({ + config: { + _: [fileNameUnderTest], + config: JSON.stringify({ rootDir: projectRoot, reporters: [] }), + findRelatedTests: true, + runInBand: true, + silent: true + }, + result: 'testResult' + }); + }); + it('should trace log a message when jest is invoked', async () => { await jestPromiseTestAdapter.run(jestConfig, projectRoot); diff --git a/packages/stryker-jest-runner/types/jest/index.d.ts b/packages/stryker-jest-runner/types/jest/index.d.ts index 9888003666..4a9b9fe152 100644 --- a/packages/stryker-jest-runner/types/jest/index.d.ts +++ b/packages/stryker-jest-runner/types/jest/index.d.ts @@ -8,6 +8,8 @@ declare namespace Jest { config: string; runInBand: boolean; silent: boolean; + findRelatedTests?: boolean; + _?: string[]; } // Taken from https://goo.gl/qHifyP, removed all stuff that we are not using @@ -18,7 +20,7 @@ declare namespace Jest { collectCoverage: boolean; verbose: boolean; testResultsProcessor: Maybe; - testEnvironment: string + testEnvironment: string; } interface RunResult { diff --git a/packages/stryker/src/Sandbox.ts b/packages/stryker/src/Sandbox.ts index 0e1784c0a8..422a1a98ac 100644 --- a/packages/stryker/src/Sandbox.ts +++ b/packages/stryker/src/Sandbox.ts @@ -44,8 +44,8 @@ export default class Sandbox { return sandbox.initialize().then(() => sandbox); } - public run(timeout: number, testHooks: string | undefined): Promise { - return this.testRunner.run({ timeout, testHooks }); + public run(timeout: number, testHooks: string | undefined, mutatedFileName?: string): Promise { + return this.testRunner.run({ timeout, testHooks, mutatedFileName }); } public dispose(): Promise { @@ -58,7 +58,7 @@ export default class Sandbox { this.log.warn(`Failed find coverage data for this mutant, running all tests. This might have an impact on performance: ${transpiledMutant.mutant.toString()}`); } await Promise.all(mutantFiles.map(mutatedFile => this.writeFileInSandbox(mutatedFile))); - const runResult = await this.run(this.calculateTimeout(transpiledMutant.mutant), this.getFilterTestsHooks(transpiledMutant.mutant)); + const runResult = await this.run(this.calculateTimeout(transpiledMutant.mutant), this.getFilterTestsHooks(transpiledMutant.mutant), this.fileMap[transpiledMutant.mutant.fileName]); await this.reset(mutantFiles); return runResult; } diff --git a/packages/stryker/test/unit/SandboxSpec.ts b/packages/stryker/test/unit/SandboxSpec.ts index 95de72c3aa..73ba2d1f5c 100644 --- a/packages/stryker/test/unit/SandboxSpec.ts +++ b/packages/stryker/test/unit/SandboxSpec.ts @@ -144,8 +144,19 @@ describe('Sandbox', () => { const sut = await Sandbox.create(options, SANDBOX_INDEX, files, null, 0, LOGGING_CONTEXT); await sut.run(231313, 'hooks'); expect(testRunner.run).to.have.been.calledWith({ + mutatedFileName: undefined, testHooks: 'hooks', - timeout: 231313 + timeout: 231313, + }); + }); + + it('should run the testRunner with mutatedFileName', async () => { + const sut = await Sandbox.create(options, SANDBOX_INDEX, files, null, 0, LOGGING_CONTEXT); + await sut.run(231313, 'hooks', 'path/to/file'); + expect(testRunner.run).to.have.been.calledWith({ + mutatedFileName: 'path/to/file', + testHooks: 'hooks', + timeout: 231313, }); }); }); @@ -196,13 +207,17 @@ describe('Sandbox', () => { expect(testFrameworkStub.filter).to.have.been.calledWith(transpiledMutant.mutant.selectedTests); }); - it('should provide the filter code as testHooks and correct timeout', async () => { + it('should provide the filter code as testHooks, correct timeout and mutatedFileName', async () => { options.timeoutMS = 1000; const overheadTimeMS = 42; const totalTimeSpend = 12; const sut = await Sandbox.create(options, SANDBOX_INDEX, files, testFrameworkStub, overheadTimeMS, LOGGING_CONTEXT); await sut.runMutant(transpiledMutant); - const expectedRunOptions = { testHooks: wrapInClosure(testFilterCodeFragment), timeout: totalTimeSpend * options.timeoutFactor + options.timeoutMS + overheadTimeMS }; + const expectedRunOptions = { + mutatedFileName: path.resolve('random-folder-3', 'file1'), + testHooks: wrapInClosure(testFilterCodeFragment), + timeout: totalTimeSpend * options.timeoutFactor + options.timeoutMS + overheadTimeMS + }; expect(testRunner.run).calledWith(expectedRunOptions); });