From bb7c02fff5296c67a71beb36cf3dbc812551364a Mon Sep 17 00:00:00 2001 From: Nico Jansen Date: Fri, 20 Oct 2017 13:50:20 +0200 Subject: [PATCH] fix(mocha framework): Select tests based on name (#413) Select tests in the `MochaTestFramework` based on it's name, rather than the order in which it is added to a test suite. * Change TestFramework interface to now filter based on test `id` and `name`. * Add `TestSelection` which represents a selected test using `id` and `name`. * Update `JasmineTestFramework` to now use the new `TestSelection` format (still based on ID). * Update `MochaTestFramework` to use the title to select tests. * Add integration tests to both `JasmineTestFramework` and `MochaTestFramework` to prevent this sort of thing from happening in the future. Fixes #249 BREAKING CHANGES: * Change api of `TestFramework`. It now provides an array of `TestSelection` objects, instead of an array of numbers with test ids. --- package.json | 1 + .../src/test_framework/TestFramework.ts | 6 +- .../src/test_framework/TestSelection.ts | 4 + .../testResources/module/useTestFramework.ts | 6 +- packages/stryker-api/test_framework.ts | 1 + packages/stryker-jasmine/.vscode/launch.json | 157 ++++----------- packages/stryker-jasmine/.vscode/tasks.json | 19 ++ packages/stryker-jasmine/package.json | 2 +- .../src/JasmineTestFramework.ts | 22 +-- .../test/integration/nestedSuiteSpec.ts | 96 ++++++++++ .../test/unit/JasmineTestFrameworkSpec.ts | 6 +- .../testResources/json-reporter.js | 13 ++ .../testResources/nested-suite.js | 16 ++ .../.vscode/launch.json | 41 ++-- .../.vscode/settings.json | 14 ++ .../.vscode/tasks.json | 19 ++ packages/stryker-mocha-framework/package.json | 2 +- .../src/MochaTestFramework.ts | 27 +-- .../test/helpers/initSourceMaps.ts | 1 + .../test/integration/nestedSuiteSpec.ts | 90 +++++++++ .../test/unit/MochaTestFrameworkSpec.ts | 12 +- .../testResources/nested-suite.js | 16 ++ .../stryker-mocha-runner/.vscode/launch.json | 54 +++--- packages/stryker/.vscode/tasks.json | 25 ++- packages/stryker/src/MutantTestMatcher.ts | 4 +- packages/stryker/src/Sandbox.ts | 19 +- packages/stryker/src/SandboxPool.ts | 2 +- packages/stryker/src/Stryker.ts | 32 ++-- packages/stryker/src/TestableMutant.ts | 9 +- .../src/coverage/CoverageInstrumenter.ts | 121 ------------ .../coverage/CoverageInstrumenterStream.ts | 49 ----- .../src/process/InitialTestExecutor.ts | 59 ++++-- .../src/process/MutationTestExecutor.ts | 8 +- .../CoverageInstrumenterTranspiler.ts | 180 ++++++++++++++++++ .../src/transpiler/TranspilerFacade.ts | 5 +- packages/stryker/src/utils/fileUtils.ts | 18 +- packages/stryker/test/helpers/producers.ts | 4 +- .../stryker/test/helpers/streamHelpers.ts | 24 --- .../test/unit/MutantTestMatcherSpec.ts | 45 +++-- packages/stryker/test/unit/SandboxPoolSpec.ts | 6 +- packages/stryker/test/unit/SandboxSpec.ts | 79 ++------ .../stryker/test/unit/TestableMutantSpec.ts | 4 +- .../unit/coverage/CoverageInstrumenterSpec.ts | 133 ------------- .../CoverageInstrumenterStreamSpec.ts | 66 ------- .../unit/process/InitialTestExecutorSpec.ts | 32 +++- .../unit/process/MutationTestExecutorSpec.ts | 15 +- .../CoverageInstrumenterTranspilerSpec.ts | 99 ++++++++++ .../unit/transpiler/TranspilerFacadeSpec.ts | 23 ++- 48 files changed, 896 insertions(+), 790 deletions(-) create mode 100644 packages/stryker-api/src/test_framework/TestSelection.ts create mode 100644 packages/stryker-jasmine/.vscode/tasks.json create mode 100644 packages/stryker-jasmine/test/integration/nestedSuiteSpec.ts create mode 100644 packages/stryker-jasmine/testResources/json-reporter.js create mode 100644 packages/stryker-jasmine/testResources/nested-suite.js create mode 100644 packages/stryker-mocha-framework/.vscode/settings.json create mode 100644 packages/stryker-mocha-framework/.vscode/tasks.json create mode 100644 packages/stryker-mocha-framework/test/helpers/initSourceMaps.ts create mode 100644 packages/stryker-mocha-framework/test/integration/nestedSuiteSpec.ts create mode 100644 packages/stryker-mocha-framework/testResources/nested-suite.js delete mode 100644 packages/stryker/src/coverage/CoverageInstrumenter.ts delete mode 100644 packages/stryker/src/coverage/CoverageInstrumenterStream.ts create mode 100644 packages/stryker/src/transpiler/CoverageInstrumenterTranspiler.ts delete mode 100644 packages/stryker/test/helpers/streamHelpers.ts delete mode 100644 packages/stryker/test/unit/coverage/CoverageInstrumenterSpec.ts delete mode 100644 packages/stryker/test/unit/coverage/CoverageInstrumenterStreamSpec.ts create mode 100644 packages/stryker/test/unit/transpiler/CoverageInstrumenterTranspilerSpec.ts diff --git a/package.json b/package.json index 25bed9855e..a87dc53ce5 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "concurrently": "^3.4.0", "execa": "^0.8.0", "glob": "^7.1.1", + "jasmine": "^2.8.0", "lerna": "^2.0.0", "link-parent-bin": "^0.1.1", "mocha": "^3.2.0", diff --git a/packages/stryker-api/src/test_framework/TestFramework.ts b/packages/stryker-api/src/test_framework/TestFramework.ts index ffb057b8f8..76bc4beed4 100644 --- a/packages/stryker-api/src/test_framework/TestFramework.ts +++ b/packages/stryker-api/src/test_framework/TestFramework.ts @@ -1,3 +1,5 @@ +import TestSelection from './TestSelection'; + /** * Represents a TestFramework which can select one or more tests to be executed. */ @@ -20,10 +22,10 @@ interface TestFramework { * will be responsible for filtering out tests with given ids. * The first test gets id 0, the second id 1, etc. * - * @param indices A list of testId's to select. + * @param selections A list indicating the tests to select. * @returns A script which, if included in the test run, will filter out the correct tests. */ - filter(ids: number[]): string; + filter(selections: TestSelection[]): string; } export default TestFramework; diff --git a/packages/stryker-api/src/test_framework/TestSelection.ts b/packages/stryker-api/src/test_framework/TestSelection.ts new file mode 100644 index 0000000000..d4ebd37154 --- /dev/null +++ b/packages/stryker-api/src/test_framework/TestSelection.ts @@ -0,0 +1,4 @@ +export default interface TestSelection { + id: number; + name: string; +} \ No newline at end of file diff --git a/packages/stryker-api/testResources/module/useTestFramework.ts b/packages/stryker-api/testResources/module/useTestFramework.ts index 8e705587a8..68c680015a 100644 --- a/packages/stryker-api/testResources/module/useTestFramework.ts +++ b/packages/stryker-api/testResources/module/useTestFramework.ts @@ -1,4 +1,4 @@ -import { TestFramework, TestFrameworkFactory, TestFrameworkSettings } from 'stryker-api/test_framework'; +import { TestFramework, TestFrameworkFactory, TestSelection, TestFrameworkSettings } from 'stryker-api/test_framework'; class TestFramework1 implements TestFramework { @@ -14,8 +14,8 @@ class TestFramework1 implements TestFramework { return ''; } - filter(ids: number[]) { - return ids.toString(); + filter(selections: TestSelection[]) { + return selections.map(selection => selection.id).toString(); } } diff --git a/packages/stryker-api/test_framework.ts b/packages/stryker-api/test_framework.ts index adc76a6823..5bb1a612ca 100644 --- a/packages/stryker-api/test_framework.ts +++ b/packages/stryker-api/test_framework.ts @@ -1,3 +1,4 @@ export {default as TestFramework} from './src/test_framework/TestFramework'; +export {default as TestSelection} from './src/test_framework/TestSelection'; export {default as TestFrameworkFactory} from './src/test_framework/TestFrameworkFactory'; export {default as TestFrameworkSettings} from './src/test_framework/TestFrameworkSettings'; diff --git a/packages/stryker-jasmine/.vscode/launch.json b/packages/stryker-jasmine/.vscode/launch.json index 5b88261cd7..a8faef94f1 100644 --- a/packages/stryker-jasmine/.vscode/launch.json +++ b/packages/stryker-jasmine/.vscode/launch.json @@ -1,123 +1,40 @@ { - "version": "0.2.0", - "configurations": [ - { - "name": "Run unit tests", - "type": "node", - "request": "launch", - "program": "${workspaceRoot}/node_modules/grunt/bin/grunt", - // "preLaunchTask": "build", - "stopOnEntry": false, - "args": [ - "mochaTest:unit" - ], - "cwd": "${workspaceRoot}/.", - "runtimeExecutable": null, - "runtimeArgs": [ - "--nolazy" - ], - "env": { - "NODE_ENV": "development" + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Unit tests", + "program": "${workspaceFolder}/../../node_modules/mocha/bin/_mocha", + "args": [ + "-u", + "tdd", + "--timeout", + "999999", + "--colors", + "${workspaceFolder}/test/helpers/**/*.js", + "${workspaceFolder}/test/unit/**/*.js" + ], + "internalConsoleOptions": "openOnSessionStart" }, - "externalConsole": false, - "sourceMaps": true, - "outDir": "${workspaceRoot}" - }, - { - "name": "Run integration tests", - "type": "node", - "request": "launch", - "program": "${workspaceRoot}/node_modules/grunt-cli/bin/grunt", - // "preLa4unchTask": "ts", - "stopOnEntry": false, - "args": [ - "mochaTest:integration" - ], - "cwd": "${workspaceRoot}/.", - "runtimeExecutable": null, - "runtimeArgs": [ - "--nolazy" - ], - "env": { - "NODE_ENV": "development" - }, - "externalConsole": false, - "sourceMaps": true, - "outDir": "${workspaceRoot}" - }, - { - "name": "Run stryker example", - "type": "node", - "request": "launch", - "program": "${workspaceRoot}/bin/stryker", - "preLaunchTask": "build", - "stopOnEntry": false, - "args": [ - "--configFile", - "testResources/sampleProject/stryker.conf.js", - "--logLevel", - "trace", - "--testFramework", - "jasmine" - ], - "cwd": "${workspaceRoot}", - "runtimeExecutable": null, - "runtimeArgs": [ - "--nolazy" - ], - "env": { - "NODE_ENV": "development" - }, - "externalConsole": false, - "sourceMaps": true, - "outDir": "${workspaceRoot}" - }, - { - "name": "Run own dog food", - "type": "node", - "request": "launch", - "program": "${workspaceRoot}/bin/stryker", - "preLaunchTask": "build", - "stopOnEntry": false, - "args": [ - "--configFile", - "stryker.conf.js", - "--logLevel", - "info" - ], - "cwd": "${workspaceRoot}", - "runtimeExecutable": null, - "runtimeArgs": [ - "--nolazy" - ], - "env": { - "NODE_ENV": "development" - }, - "externalConsole": false, - "sourceMaps": true, - "outDir": "${workspaceRoot}" - }, - { - "name": "Run stryker help", - "type": "node", - "request": "launch", - "program": "${workspaceRoot}/src/Stryker.js", - "preLaunchTask": "build", - "stopOnEntry": false, - "args": [ - "--help" - ], - "cwd": "${workspaceRoot}/.", - "runtimeExecutable": null, - "runtimeArgs": [ - "--nolazy" - ], - "env": { - "NODE_ENV": "development" - }, - "externalConsole": false, - "sourceMaps": true, - "outDir": "${workspaceRoot}" - } - ] + { + "type": "node", + "request": "launch", + "name": "Integration tests", + "program": "${workspaceFolder}/../../node_modules/mocha/bin/_mocha", + "args": [ + "-u", + "tdd", + "--timeout", + "999999", + "--colors", + "${workspaceFolder}/test/helpers/**/*.js", + "${workspaceFolder}/test/integration/**/*.js" + ], + "internalConsoleOptions": "openOnSessionStart" + } + ] } \ No newline at end of file diff --git a/packages/stryker-jasmine/.vscode/tasks.json b/packages/stryker-jasmine/.vscode/tasks.json new file mode 100644 index 0000000000..fe88541ac1 --- /dev/null +++ b/packages/stryker-jasmine/.vscode/tasks.json @@ -0,0 +1,19 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "type": "typescript", + "tsconfig": "tsconfig.json", + "option": "watch", + "problemMatcher": [ + "$tsc-watch" + ], + "group": { + "kind": "build", + "isDefault": true + } + } + ] +} \ No newline at end of file diff --git a/packages/stryker-jasmine/package.json b/packages/stryker-jasmine/package.json index e25aa68d0e..77a3933838 100644 --- a/packages/stryker-jasmine/package.json +++ b/packages/stryker-jasmine/package.json @@ -9,7 +9,7 @@ "prebuild": "npm run clean", "build": "tsc -p .", "postbuild": "tslint -p tsconfig.json", - "test": "nyc --reporter=html --report-dir=reports/coverage --check-coverage --lines 100 --functions 100 --branches 100 mocha \"test/**/*.js\"" + "test": "nyc --reporter=html --report-dir=reports/coverage --check-coverage --lines 100 --functions 100 --branches 100 mocha \"test/helpers/**/*.js\" \"test/unit/**/*.js\" \"test/integration/**/*.js\"" }, "repository": { "type": "git", diff --git a/packages/stryker-jasmine/src/JasmineTestFramework.ts b/packages/stryker-jasmine/src/JasmineTestFramework.ts index d8c3751d12..4bdf4edb0d 100644 --- a/packages/stryker-jasmine/src/JasmineTestFramework.ts +++ b/packages/stryker-jasmine/src/JasmineTestFramework.ts @@ -1,8 +1,8 @@ -import { TestFramework, TestFrameworkSettings } from 'stryker-api/test_framework'; +import { TestFramework, TestSelection } from 'stryker-api/test_framework'; export default class JasmineTestFramework implements TestFramework { - constructor(settings: TestFrameworkSettings) { + constructor() { } /** @@ -31,24 +31,12 @@ export default class JasmineTestFramework implements TestFramework { });`; } - /** - * Creates a code fragment which, in included in a test run, - * will be responsible for filtering out tests with given ids. - * The first test gets id 0, the second id 1, etc. - * - * @param indices A list of testId's to select. - * @returns A script which, if included in the test run, will filter out the correct tests. - */ - filter(ids: number[]): string { + filter(testSelections: TestSelection[]): string { + const ids = testSelections.map(selection => selection.id); return ` var currentTestId = 0; jasmine.getEnv().specFilter = function (spec) { - var filterOut = false; - if(${JSON.stringify(ids)}.indexOf(currentTestId) >= 0){ - filterOut = true; - } - currentTestId++; - return filterOut; + return ${JSON.stringify(ids)}.indexOf(currentTestId++) !== -1; }`; } } \ No newline at end of file diff --git a/packages/stryker-jasmine/test/integration/nestedSuiteSpec.ts b/packages/stryker-jasmine/test/integration/nestedSuiteSpec.ts new file mode 100644 index 0000000000..4f0b12592c --- /dev/null +++ b/packages/stryker-jasmine/test/integration/nestedSuiteSpec.ts @@ -0,0 +1,96 @@ +import * as path from 'path'; +import * as execa from 'execa'; +import { expect } from 'chai'; +import { TestSelection } from 'stryker-api/test_framework'; +import JasmineTestFramework from '../../src/JasmineTestFramework'; +import * as fs from 'fs'; +import * as rimraf from 'rimraf'; + +interface JasmineTest { + id: string; + description: string; + fullName: string; + failedExpectations: any[]; + passedExpectations: any[]; + status: string; +} + +describe('Selecting tests with nested suites', function () { + + this.timeout(10000); + let sut: JasmineTestFramework; + let testSelections: TestSelection[]; + const jsonReporterFile = path.resolve(__dirname, '..', '..', 'testResources', 'json-reporter.js'); + const nestedSuiteFile = path.resolve(__dirname, '..', '..', 'testResources', 'nested-suite.js'); + const selectTestFile = path.join(__dirname, '..', '..', 'testResources', '__filterSpecs.js'); + + beforeEach(() => { + sut = new JasmineTestFramework(); + testSelections = [ + { id: 0, name: 'outer test 1' }, + { id: 1, name: 'outer test 2' }, + { id: 2, name: 'outer inner test 3' } + ]; + }); + + afterEach(() => { + rimraf.sync(selectTestFile); + }); + + it('should run all tests in expected order when running all tests', () => { + const result = execJasmine(nestedSuiteFile); + expect(result.map(test => test.fullName)).deep.eq(['outer test 1', 'outer inner test 2', 'outer test 3']); + }); + + it('should only run test 1 if filtered on index 0', () => { + filter([0]); + const result = execJasmine(selectTestFile, nestedSuiteFile); + expect(result).lengthOf(3); + expect(result[0].status).eq('passed'); + expect(result[1].status).eq('disabled'); + expect(result[2].status).eq('disabled'); + expect(result[0].fullName).eq('outer test 1'); + }); + + it('should only run test 2 if filtered on index 1', () => { + filter([1]); + const result = execJasmine(selectTestFile, nestedSuiteFile); + expect(result).lengthOf(3); + expect(result[0].status).eq('disabled'); + expect(result[1].status).eq('passed'); + expect(result[2].status).eq('disabled'); + expect(result[1].fullName).eq('outer inner test 2'); + }); + + it('should only run test 3 if filtered on index 2', () => { + filter([2]); + const result = execJasmine(selectTestFile, nestedSuiteFile); + expect(result).lengthOf(3); + expect(result[0].status).eq('disabled'); + expect(result[1].status).eq('disabled'); + expect(result[2].status).eq('passed'); + expect(result[2].fullName).eq('outer test 3'); + }); + + it('should only run tests 1 and 3 if filtered on indices 0 and 2', () => { + filter([0, 2]); + const result = execJasmine(selectTestFile, nestedSuiteFile); + expect(result).lengthOf(3); + expect(result[0].status).eq('passed'); + expect(result[1].status).eq('disabled'); + expect(result[2].status).eq('passed'); + expect(result[0].fullName).eq('outer test 1'); + expect(result[2].fullName).eq('outer test 3'); + }); + + function filter(testIds: number[]) { + const selections = testIds.map(id => testSelections[id]); + const filterFn = `(function (window) {${sut.filter(selections)}})(global);`; + fs.writeFileSync(selectTestFile, filterFn, 'utf8'); + } + + function execJasmine(...files: string[]): JasmineTest[] { + const execResult = execa.sync('jasmine', [jsonReporterFile, ...files]); + return JSON.parse(execResult.stdout); + } +}); \ No newline at end of file diff --git a/packages/stryker-jasmine/test/unit/JasmineTestFrameworkSpec.ts b/packages/stryker-jasmine/test/unit/JasmineTestFrameworkSpec.ts index 98919578e3..1d6cec22c6 100644 --- a/packages/stryker-jasmine/test/unit/JasmineTestFrameworkSpec.ts +++ b/packages/stryker-jasmine/test/unit/JasmineTestFrameworkSpec.ts @@ -3,7 +3,7 @@ import JasmineTestFramework from '../../src/JasmineTestFramework'; describe('JasmineTestFramework', () => { let sut: JasmineTestFramework; - beforeEach(() => sut = new JasmineTestFramework({ options: {} })); + beforeEach(() => sut = new JasmineTestFramework()); describe('beforeEach()', () => { it('should result in a specStarted reporter hook', () => @@ -17,8 +17,8 @@ describe('JasmineTestFramework', () => { describe('filter()', () => { it('should result in a specFilter of jasmine it\'s', () => - expect(sut.filter([5, 8])) + expect(sut.filter([{ id: 5, name: 'test five' }, { id: 8, name: 'test eight' }])) .to.contain('jasmine.getEnv().specFilter = function (spec)') - .and.to.contain('if([5,8].indexOf(currentTestId) >= 0){')); + .and.to.contain('return [5,8].indexOf(currentTestId++) !== -1;')); }); }); \ No newline at end of file diff --git a/packages/stryker-jasmine/testResources/json-reporter.js b/packages/stryker-jasmine/testResources/json-reporter.js new file mode 100644 index 0000000000..a73ad2969d --- /dev/null +++ b/packages/stryker-jasmine/testResources/json-reporter.js @@ -0,0 +1,13 @@ + +const results = []; +jasmine.getEnv().clearReporters(); +jasmine.getEnv().addReporter({ + + specDone(result) { + results.push(result); + }, + + jasmineDone() { + console.log(JSON.stringify(results)); + } +}); \ No newline at end of file diff --git a/packages/stryker-jasmine/testResources/nested-suite.js b/packages/stryker-jasmine/testResources/nested-suite.js new file mode 100644 index 0000000000..4a270b0301 --- /dev/null +++ b/packages/stryker-jasmine/testResources/nested-suite.js @@ -0,0 +1,16 @@ +describe('outer', () => { + + it('test 1', () => { + + }); + + describe('inner', () => { + it('test 2', () => { + + }); + }); + + it('test 3', () => { + + }); +}); \ No newline at end of file diff --git a/packages/stryker-mocha-framework/.vscode/launch.json b/packages/stryker-mocha-framework/.vscode/launch.json index 9cb5fc997f..f8e9db187e 100644 --- a/packages/stryker-mocha-framework/.vscode/launch.json +++ b/packages/stryker-mocha-framework/.vscode/launch.json @@ -1,17 +1,28 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "type": "node", - "request": "launch", - "name": "Launch Program", - "program": "${workspaceFolder}\\src\\index.js", - "outFiles": [ - "${workspaceFolder}/**/*.js" - ] - } - ] + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Unit tests", + "program": "${workspaceFolder}/../../node_modules/mocha/bin/mocha", + "args": [ + "test/helpers/**/*.js", + "test/unit/**/*.js" + ] + }, + { + "type": "node", + "request": "launch", + "name": "Integration tests", + "program": "${workspaceFolder}/../../node_modules/mocha/bin/_mocha", + "args": [ + "test/helpers/**/*.js", + "test/integration/**/*.js" + ] + } + ] } \ No newline at end of file diff --git a/packages/stryker-mocha-framework/.vscode/settings.json b/packages/stryker-mocha-framework/.vscode/settings.json new file mode 100644 index 0000000000..7d0eb1f0dd --- /dev/null +++ b/packages/stryker-mocha-framework/.vscode/settings.json @@ -0,0 +1,14 @@ +{ + "files.exclude": { + "**/.git": true, + "**/.svn": true, + "**/.hg": true, + "**/CVS": true, + "**/.DS_Store": true, + "**/*.map": true, + "**/*.d.ts": true, + "**/*.js": { + "when": "$(basename).ts" + } + } +} \ No newline at end of file diff --git a/packages/stryker-mocha-framework/.vscode/tasks.json b/packages/stryker-mocha-framework/.vscode/tasks.json new file mode 100644 index 0000000000..fe88541ac1 --- /dev/null +++ b/packages/stryker-mocha-framework/.vscode/tasks.json @@ -0,0 +1,19 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "type": "typescript", + "tsconfig": "tsconfig.json", + "option": "watch", + "problemMatcher": [ + "$tsc-watch" + ], + "group": { + "kind": "build", + "isDefault": true + } + } + ] +} \ No newline at end of file diff --git a/packages/stryker-mocha-framework/package.json b/packages/stryker-mocha-framework/package.json index 41278b33ec..03f8802ad1 100644 --- a/packages/stryker-mocha-framework/package.json +++ b/packages/stryker-mocha-framework/package.json @@ -9,7 +9,7 @@ "prebuild": "npm run clean", "build": "tsc -p .", "postbuild": "tslint -p tsconfig.json", - "test": "nyc --reporter=html --report-dir=reports/coverage --check-coverage --lines 100 --functions 100 --branches 100 mocha \"test/**/*.js\"" + "test": "nyc --include src/**/*.js --reporter=html --report-dir=reports/coverage --check-coverage --lines 100 --functions 100 --branches 100 mocha \"test/helpers/**/*.js\" \"test/unit/**/*.js\" \"test/integration/**/*.js\"" }, "repository": { "type": "git", diff --git a/packages/stryker-mocha-framework/src/MochaTestFramework.ts b/packages/stryker-mocha-framework/src/MochaTestFramework.ts index 2a176f8ec8..bf13161b6d 100644 --- a/packages/stryker-mocha-framework/src/MochaTestFramework.ts +++ b/packages/stryker-mocha-framework/src/MochaTestFramework.ts @@ -1,4 +1,4 @@ -import { TestFramework } from 'stryker-api/test_framework'; +import { TestFramework, TestSelection } from 'stryker-api/test_framework'; export default class MochaTestFramework implements TestFramework { @@ -14,22 +14,23 @@ export default class MochaTestFramework implements TestFramework { });`; } - filter(testIds: number[]) { - return ` - var mocha = window.mocha || require('mocha'); + filter(testSelections: TestSelection[]) { + const selectedTestNames = testSelections.map(selection => selection.name); + return `var Mocha = window.Mocha || require('mocha'); + var selectedTestNames = ${JSON.stringify(selectedTestNames)}; if (window.____mochaAddTest) { - mocha.Suite.prototype.addTest = window.____mochaAddTest; + Mocha.Suite.prototype.addTest = window.____mochaAddTest; } else { - window.____mochaAddTest = mocha.Suite.prototype.addTest + window.____mochaAddTest = Mocha.Suite.prototype.addTest } - var current = 0; - var realAddTest = mocha.Suite.prototype.addTest; - mocha.Suite.prototype.addTest = function () { - if (${JSON.stringify(testIds)}.indexOf(current) > -1) { + var realAddTest = Mocha.Suite.prototype.addTest; + + Mocha.Suite.prototype.addTest = function (test) { + // Only actually add the tests with the expected names + var name = this.fullTitle() + ' ' + test.title; + if(selectedTestNames.indexOf(name) !== -1) { realAddTest.apply(this, arguments); } - current++; - }; - `; + };`; } } diff --git a/packages/stryker-mocha-framework/test/helpers/initSourceMaps.ts b/packages/stryker-mocha-framework/test/helpers/initSourceMaps.ts new file mode 100644 index 0000000000..f1231194f5 --- /dev/null +++ b/packages/stryker-mocha-framework/test/helpers/initSourceMaps.ts @@ -0,0 +1 @@ +import 'source-map-support/register'; \ No newline at end of file diff --git a/packages/stryker-mocha-framework/test/integration/nestedSuiteSpec.ts b/packages/stryker-mocha-framework/test/integration/nestedSuiteSpec.ts new file mode 100644 index 0000000000..d758cfa93c --- /dev/null +++ b/packages/stryker-mocha-framework/test/integration/nestedSuiteSpec.ts @@ -0,0 +1,90 @@ +import * as path from 'path'; +import * as execa from 'execa'; +import { expect } from 'chai'; +import { TestSelection } from 'stryker-api/test_framework'; +import MochaTestFramework from '../../src/MochaTestFramework'; +import * as fs from 'fs'; +import * as rimraf from 'rimraf'; + +interface MochaTestRunResult { + tests: MochaTest[]; + pending: MochaTest[]; + failure: MochaTest[]; + passes: MochaTest[]; +} + +interface MochaTest { + title: string; + fullTitle: string; +} + +// See https://github.com/stryker-mutator/stryker/issues/249 +describe('Selecting tests with nested suites', function () { + + this.timeout(10000); + let sut: MochaTestFramework; + const nestedSuiteFile = path.resolve(__dirname, '..', '..', 'testResources', 'nested-suite.js'); + const selectTestFile = path.join(__dirname, '..', '..', 'testResources', '__filterSpecs.js'); + const testSelections: ReadonlyArray> = [ + { id: 0, name: 'outer test 1' }, + { id: 1, name: 'outer test 2' }, + { id: 2, name: 'outer inner test 3' } + ]; + + beforeEach(() => { + sut = new MochaTestFramework(); + }); + + afterEach(() => { + rimraf.sync(selectTestFile); + }); + + it('should run all tests in expected order when running all tests', () => { + const result = execMocha(nestedSuiteFile); + expect(result.tests.map(test => test.fullTitle)).deep.eq(['outer test 1', 'outer test 2', 'outer inner test 3']); + }); + + it('should only run test 1 if filtered on index 0', () => { + filter([0]); + const result = execMocha(selectTestFile, nestedSuiteFile); + expect(result.tests).lengthOf(1); + expect(result.passes).lengthOf(1); + expect(result.passes[0].fullTitle).eq('outer test 1'); + }); + + it('should only run test 2 if filtered on index 1', () => { + filter([1]); + const result = execMocha(selectTestFile, nestedSuiteFile); + expect(result.tests).lengthOf(1); + expect(result.passes).lengthOf(1); + expect(result.passes[0].fullTitle).eq('outer test 2'); + }); + + it('should only run test 3 if filtered on index 2', () => { + filter([2]); + const result = execMocha(selectTestFile, nestedSuiteFile); + expect(result.tests).lengthOf(1); + expect(result.passes).lengthOf(1); + expect(result.passes[0].fullTitle).eq('outer inner test 3'); + }); + + it('should run tests 1 and 3 if filtered on indices 0 and 2', () => { + filter([0, 2]); + const result = execMocha(selectTestFile, nestedSuiteFile); + expect(result.tests).lengthOf(2); + expect(result.passes).lengthOf(2); + expect(result.passes[0].fullTitle).eq('outer test 1'); + expect(result.passes[1].fullTitle).eq('outer inner test 3'); + }); + + function filter(testIds: number[]) { + const selections = testIds.map(id => testSelections[id]); + const filterFn = `(function (window) {${sut.filter(selections)}})(global);`; + fs.writeFileSync(selectTestFile, filterFn, 'utf8'); + } + + function execMocha(...files: string[]) { + const execResult = execa.sync('mocha', ['--reporter', 'json', ...files]); + return JSON.parse(execResult.stdout) as MochaTestRunResult; + } +}); \ No newline at end of file diff --git a/packages/stryker-mocha-framework/test/unit/MochaTestFrameworkSpec.ts b/packages/stryker-mocha-framework/test/unit/MochaTestFrameworkSpec.ts index 9e1cde4444..f41f898469 100644 --- a/packages/stryker-mocha-framework/test/unit/MochaTestFrameworkSpec.ts +++ b/packages/stryker-mocha-framework/test/unit/MochaTestFrameworkSpec.ts @@ -16,10 +16,12 @@ describe('MochaTestFramework', () => { }); describe('filter()', () => { - it('should result in a filtering of mocha it\'s', () => - expect(sut.filter([5, 8])) - .to.contain('var realAddTest = mocha.Suite.prototype.addTest;') - .and.to.contain('if ([5,8].indexOf(current) > -1)') - .and.to.contain('realAddTest.apply(this, arguments);')); + it('should result in a filtering of mocha it\'s', () => { + expect(sut.filter([{ id: 5, name: 'test five' }, { id: 8, name: 'test eight' }])) + .to.contain('var realAddTest = Mocha.Suite.prototype.addTest;') + .and.to.contain('selectedTestNames = ["test five","test eight"];') + .and.to.contain('if(selectedTestNames.indexOf(name) !== -1)') + .and.to.contain('realAddTest.apply(this, arguments);'); + }); }); }); \ No newline at end of file diff --git a/packages/stryker-mocha-framework/testResources/nested-suite.js b/packages/stryker-mocha-framework/testResources/nested-suite.js new file mode 100644 index 0000000000..791e07920c --- /dev/null +++ b/packages/stryker-mocha-framework/testResources/nested-suite.js @@ -0,0 +1,16 @@ +describe('outer', () => { + + it('test 1', () => { + + }); + + describe('inner', () => { + it('test 3', () => { + + }); + }); + + it('test 2', () => { + + }); +}); \ No newline at end of file diff --git a/packages/stryker-mocha-runner/.vscode/launch.json b/packages/stryker-mocha-runner/.vscode/launch.json index 7de952f0c0..bfc69450ae 100644 --- a/packages/stryker-mocha-runner/.vscode/launch.json +++ b/packages/stryker-mocha-runner/.vscode/launch.json @@ -1,48 +1,40 @@ { + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ { - "name": "Run unit tests", "type": "node", "request": "launch", - "program": "${workspaceRoot}/node_modules/grunt-cli/bin/grunt", - "stopOnEntry": false, + "name": "Unit tests", + "program": "${workspaceFolder}/../../node_modules/mocha/bin/_mocha", "args": [ - "mochaTest:unit" + "-u", + "tdd", + "--timeout", + "999999", + "--colors", + "${workspaceFolder}/test/helpers/**/*.js", + "${workspaceFolder}/test/unit/**/*.js" ], - "cwd": "${workspaceRoot}/.", - "runtimeExecutable": null, - "runtimeArgs": [ - "--nolazy" - ], - "env": { - "NODE_ENV": "development" - }, - "externalConsole": false, - "sourceMaps": true, - "outDir": "${workspaceRoot}" + "internalConsoleOptions": "openOnSessionStart" }, { - "name": "Run integration tests", "type": "node", "request": "launch", - "program": "${workspaceRoot}/node_modules/grunt-cli/bin/grunt", - // "preLaunchTask": "build", - "stopOnEntry": false, + "name": "Integration tests", + "program": "${workspaceFolder}/../../node_modules/mocha/bin/_mocha", "args": [ - "integration" - ], - "cwd": "${workspaceRoot}/.", - "runtimeExecutable": null, - "runtimeArgs": [ - "--nolazy" + "-u", + "tdd", + "--timeout", + "999999", + "--colors", + "${workspaceFolder}/test/helpers/**/*.js", + "${workspaceFolder}/test/integration/**/*.js" ], - "env": { - "NODE_ENV": "development" - }, - "externalConsole": false, - "sourceMaps": true, - "outDir": "${workspaceRoot}" + "internalConsoleOptions": "openOnSessionStart" } ] } \ No newline at end of file diff --git a/packages/stryker/.vscode/tasks.json b/packages/stryker/.vscode/tasks.json index 86ffa557d6..efca969da5 100644 --- a/packages/stryker/.vscode/tasks.json +++ b/packages/stryker/.vscode/tasks.json @@ -1,11 +1,18 @@ { - // See https://go.microsoft.com/fwlink/?LinkId=733558 - // for the documentation about the tasks.json format - "version": "0.1.0", - "command": "npm", - "isShellCommand": true, - "args": ["start"], - "showOutput": "silent", - "isBackground": true, - "problemMatcher": "$tsc-watch" + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "taskName": "tsc-watch", + "type": "shell", + "command": "npm start", + "problemMatcher": "$tsc-watch", + "isBackground": true, + "group": { + "kind": "build", + "isDefault": true + } + } + ] } \ No newline at end of file diff --git a/packages/stryker/src/MutantTestMatcher.ts b/packages/stryker/src/MutantTestMatcher.ts index ec9dcc6256..a1f5b42463 100644 --- a/packages/stryker/src/MutantTestMatcher.ts +++ b/packages/stryker/src/MutantTestMatcher.ts @@ -6,7 +6,7 @@ import { MatchedMutant } from 'stryker-api/report'; import { Mutant } from 'stryker-api/mutant'; import TestableMutant from './TestableMutant'; import StrictReporter from './reporters/StrictReporter'; -import { StatementMapDictionary } from './coverage/CoverageInstrumenter'; +import { StatementMapDictionary } from './transpiler/CoverageInstrumenterTranspiler'; import { filterEmpty } from './utils/objectUtils'; import SourceFile from './SourceFile'; @@ -96,7 +96,7 @@ export default class MutantTestMatcher { private mapMutantOnMatchedMutant(testableMutant: TestableMutant): MatchedMutant { const matchedMutant = _.cloneDeep({ mutatorName: testableMutant.mutant.mutatorName, - scopedTestIds: testableMutant.scopedTestIds, + scopedTestIds: testableMutant.selectedTests.map(testSelection => testSelection.id), timeSpentScopedTests: testableMutant.timeSpentScopedTests, fileName: testableMutant.mutant.fileName, replacement: testableMutant.mutant.replacement diff --git a/packages/stryker/src/Sandbox.ts b/packages/stryker/src/Sandbox.ts index 0e032c68e1..9f151aa8a9 100644 --- a/packages/stryker/src/Sandbox.ts +++ b/packages/stryker/src/Sandbox.ts @@ -11,7 +11,6 @@ import ResilientTestRunnerFactory from './isolated-runner/ResilientTestRunnerFac import IsolatedRunnerOptions from './isolated-runner/IsolatedRunnerOptions'; import { TempFolder } from './utils/TempFolder'; import * as fileUtils from './utils/fileUtils'; -import CoverageInstrumenter from './coverage/CoverageInstrumenter'; import TestableMutant from './TestableMutant'; import TranspiledMutant from './TranspiledMutant'; @@ -28,7 +27,7 @@ export default class Sandbox { private workingFolder: string; private testHooksFile = path.resolve('___testHooksForStryker.js'); - private constructor(private options: Config, private index: number, files: ReadonlyArray, private testFramework: TestFramework | null, private coverageInstrumenter: CoverageInstrumenter | null) { + private constructor(private options: Config, private index: number, files: ReadonlyArray, private testFramework: TestFramework | null) { this.workingFolder = TempFolder.instance().createRandomFolder('sandbox'); this.log.debug('Creating a sandbox for files in %s', this.workingFolder); this.files = files.slice(); // Create a copy @@ -36,7 +35,7 @@ export default class Sandbox { this.testHooksFile = path.resolve('___testHooksForStryker.js'); this.files.unshift({ name: this.testHooksFile, - content: coverageInstrumenter && coverageInstrumenter.hooksForTestRun() || '', + content: '', mutated: false, included: true, transpiled: false, @@ -50,9 +49,9 @@ export default class Sandbox { return this.initializeTestRunner(); } - public static create(options: Config, index: number, files: ReadonlyArray, testFramework: TestFramework | null, coverageInstrumenter: CoverageInstrumenter | null) + public static create(options: Config, index: number, files: ReadonlyArray, testFramework: TestFramework | null) : Promise { - const sandbox = new Sandbox(options, index, files, testFramework, coverageInstrumenter); + const sandbox = new Sandbox(options, index, files, testFramework); return sandbox.initialize().then(() => sandbox); } @@ -108,14 +107,12 @@ export default class Sandbox { return Promise.resolve(); default: const cwd = process.cwd(); - const relativePath = file.name.substr(cwd.length); - const folderName = this.workingFolder + path.dirname(relativePath); + const relativePath = path.relative(cwd, file.name); + const folderName = path.join(this.workingFolder, path.dirname(relativePath)); mkdirp.sync(folderName); const targetFile = path.join(folderName, path.basename(relativePath)); this.fileMap[file.name] = targetFile; - const instrumentingStream = this.coverageInstrumenter ? - this.coverageInstrumenter.instrumenterStreamForFile(file) : null; - return fileUtils.writeFile(targetFile, file.content, instrumentingStream); + return fileUtils.writeFile(targetFile, file.content); } } @@ -145,7 +142,7 @@ export default class Sandbox { private filterTests(mutant: TestableMutant) { if (this.testFramework) { - let fileContent = wrapInClosure(this.testFramework.filter(mutant.scopedTestIds)); + let fileContent = wrapInClosure(this.testFramework.filter(mutant.selectedTests)); return fileUtils.writeFile(this.fileMap[this.testHooksFile], fileContent); } else { return Promise.resolve(void 0); diff --git a/packages/stryker/src/SandboxPool.ts b/packages/stryker/src/SandboxPool.ts index f69e5cf4d3..4383ff7d27 100644 --- a/packages/stryker/src/SandboxPool.ts +++ b/packages/stryker/src/SandboxPool.ts @@ -32,7 +32,7 @@ export default class SandboxPool { this.log.info(`Creating ${numConcurrentRunners} test runners (based on ${numConcurrentRunnersSource})`); const sandboxes = Observable.range(0, numConcurrentRunners) - .flatMap(n => this.registerSandbox(Sandbox.create(this.options, n, this.initialFiles, this.testFramework, null))); + .flatMap(n => this.registerSandbox(Sandbox.create(this.options, n, this.initialFiles, this.testFramework))); return sandboxes; } diff --git a/packages/stryker/src/Stryker.ts b/packages/stryker/src/Stryker.ts index cc6def1834..e02839bb5b 100644 --- a/packages/stryker/src/Stryker.ts +++ b/packages/stryker/src/Stryker.ts @@ -3,7 +3,6 @@ import { StrykerOptions, File } from 'stryker-api/core'; import { MutantResult } from 'stryker-api/report'; import { TestFramework } from 'stryker-api/test_framework'; import ReporterOrchestrator from './ReporterOrchestrator'; -import { RunResult } from 'stryker-api/test_runner'; import TestFrameworkOrchestrator from './TestFrameworkOrchestrator'; import MutantTestMatcher from './MutantTestMatcher'; import InputFileResolver from './InputFileResolver'; @@ -11,14 +10,13 @@ import ConfigReader from './ConfigReader'; import PluginLoader from './PluginLoader'; import ScoreResultCalculator from './ScoreResultCalculator'; import ConfigValidator from './ConfigValidator'; -import CoverageInstrumenter from './coverage/CoverageInstrumenter'; import { freezeRecursively, isPromise } from './utils/objectUtils'; import { TempFolder } from './utils/TempFolder'; import * as log4js from 'log4js'; import Timer from './utils/Timer'; import StrictReporter from './reporters/StrictReporter'; import MutatorFacade from './MutatorFacade'; -import InitialTestExecutor from './process/InitialTestExecutor'; +import InitialTestExecutor, { InitialTestRunResult } from './process/InitialTestExecutor'; import MutationTestExecutor from './process/MutationTestExecutor'; export default class Stryker { @@ -27,7 +25,6 @@ export default class Stryker { private timer = new Timer(); private reporter: StrictReporter; private testFramework: TestFramework | null; - private coverageInstrumenter: CoverageInstrumenter; private readonly log = log4js.getLogger(Stryker.name); /** @@ -45,19 +42,18 @@ export default class Stryker { this.freezeConfig(); this.reporter = new ReporterOrchestrator(this.config).createBroadcastReporter(); this.testFramework = new TestFrameworkOrchestrator(this.config).determineTestFramework(); - this.coverageInstrumenter = new CoverageInstrumenter(this.config.coverageAnalysis, this.testFramework); new ConfigValidator(this.config, this.testFramework).validate(); } async runMutationTest(): Promise { this.timer.reset(); const inputFiles = await new InputFileResolver(this.config.mutate, this.config.files, this.reporter).resolve(); - TempFolder.instance().initialize(); + TempFolder.instance().initialize(); const initialTestRunProcess = this.createInitialTestRunner(inputFiles); - const { runResult, transpiledFiles } = await initialTestRunProcess.run(); - const testableMutants = await this.mutate(inputFiles, runResult); - if (runResult.tests.length && testableMutants.length) { - const mutationTestExecutor = this.createMutationTester(inputFiles, transpiledFiles); + const initialTestRunResult = await initialTestRunProcess.run(); + const testableMutants = await this.mutate(inputFiles, initialTestRunResult); + if (initialTestRunResult.runResult.tests.length && testableMutants.length) { + const mutationTestExecutor = this.createMutationTester(inputFiles); const mutantResults = await mutationTestExecutor.run(testableMutants); this.reportScore(mutantResults); await this.wrapUpReporter(); @@ -69,7 +65,7 @@ export default class Stryker { } } - private mutate(inputFiles: File[], runResult: RunResult) { + private mutate(inputFiles: File[], initialTestRunResult: InitialTestRunResult) { const mutator = new MutatorFacade(this.config); const mutants = mutator.mutate(inputFiles); if (mutants.length) { @@ -77,7 +73,13 @@ export default class Stryker { } else { this.log.info('It\'s a mutant-free world, nothing to test.'); } - const mutantRunResultMatcher = new MutantTestMatcher(mutants, inputFiles, runResult, this.coverageInstrumenter.retrieveStatementMapsPerFile(), this.config, this.reporter); + const mutantRunResultMatcher = new MutantTestMatcher( + mutants, + inputFiles, + initialTestRunResult.runResult, + initialTestRunResult.statementMaps, + this.config, + this.reporter); return mutantRunResultMatcher.matchWithMutants(); } @@ -117,12 +119,12 @@ export default class Stryker { log4js.setGlobalLogLevel(this.config.logLevel); } - private createMutationTester(inputFiles: File[], transpiledFiles: File[]) { - return new MutationTestExecutor(this.config, inputFiles, transpiledFiles, this.testFramework, this.reporter); + private createMutationTester(inputFiles: File[]) { + return new MutationTestExecutor(this.config, inputFiles, this.testFramework, this.reporter); } private createInitialTestRunner(inputFiles: File[]) { - return new InitialTestExecutor(this.config, inputFiles, this.coverageInstrumenter, this.testFramework, this.timer); + return new InitialTestExecutor(this.config, inputFiles, this.testFramework, this.timer); } private reportScore(mutantResults: MutantResult[]) { diff --git a/packages/stryker/src/TestableMutant.ts b/packages/stryker/src/TestableMutant.ts index 5f358da3a1..adda101e12 100644 --- a/packages/stryker/src/TestableMutant.ts +++ b/packages/stryker/src/TestableMutant.ts @@ -4,17 +4,18 @@ import { Mutant } from 'stryker-api/mutant'; import SourceFile, { isLineBreak } from './SourceFile'; import { MutantStatus, MutantResult } from 'stryker-api/report'; import { freezeRecursively } from './utils/objectUtils'; +import { TestSelection } from 'stryker-api/test_framework'; export default class TestableMutant { - private _scopedTestIds: number[] = []; + private _selectedTests: TestSelection[] = []; public specsRan: string[] = []; private _timeSpentScopedTests = 0; private _location: Location; - get scopedTestIds(): number[] { - return this._scopedTestIds; + get selectedTests(): TestSelection[] { + return this._selectedTests; } get timeSpentScopedTests() { @@ -63,7 +64,7 @@ export default class TestableMutant { } public addTestResult(index: number, testResult: TestResult) { - this._scopedTestIds.push(index); + this._selectedTests.push({ id: index, name: testResult.name }); this._timeSpentScopedTests += testResult.timeSpentMs; } diff --git a/packages/stryker/src/coverage/CoverageInstrumenter.ts b/packages/stryker/src/coverage/CoverageInstrumenter.ts deleted file mode 100644 index 86d78344f1..0000000000 --- a/packages/stryker/src/coverage/CoverageInstrumenter.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { getLogger } from 'log4js'; -import { FileDescriptor } from 'stryker-api/core'; -import { PassThrough } from 'stream'; -import { StatementMap } from 'stryker-api/test_runner'; -import { TestFramework } from 'stryker-api/test_framework'; -import { wrapInClosure } from '../utils/objectUtils'; -import CoverageInstrumenterStream from './CoverageInstrumenterStream'; - -export interface StatementMapDictionary { - [file: string]: StatementMap; -} - -const COVERAGE_CURRENT_TEST_VARIABLE_NAME = '__strykerCoverageCurrentTest__'; - -/** - * Represents the CoverageInstrumenter - * Responsible for managing the instrumentation of all files to be mutated. - * In case of `perTest` coverageAnalysis it will hook into the test framework to accomplish that. - */ -export default class CoverageInstrumenter { - - private readonly log = getLogger(CoverageInstrumenter.name); - - private readonly coverageInstrumenterStreamPerFile: { [fileName: string]: CoverageInstrumenterStream } = Object.create(null); - - constructor(private coverageAnalysis: 'all' | 'off' | 'perTest', private testFramework: TestFramework | null) { } - - public instrumenterStreamForFile(file: FileDescriptor): NodeJS.ReadWriteStream { - if (file.mutated) { - /* - Coverage variable *must* have the name '__coverage__'. Only that variable - is reported back to the TestRunner process when using one of the karma - test framework adapters (karma-jasmine, karma-mocha, ...). - - However, when coverageAnalysis is 'perTest' we don't choose that variable name right away, - because we need that variable to hold all coverage results per test. Instead, we use __strykerCoverageCurrentTest__ - and after each test copy over the value of that current test to the global coverage object __coverage__ - */ - switch (this.coverageAnalysis) { - case 'all': - return this.createStreamForFile('__coverage__', file.name); - case 'perTest': - return this.createStreamForFile(COVERAGE_CURRENT_TEST_VARIABLE_NAME, file.name); - } - } - // By default, do not instrument for code coverage - return new PassThrough(); - } - - public hooksForTestRun(): string | null { - if (this.testFramework && this.coverageAnalysis === 'perTest') { - this.log.debug(`Adding test hooks file for coverageAnalysis "perTest"`); - return wrapInClosure(` - var id = 0; - window.__coverage__ = globalCoverage = { deviations: {} }; - ${this.testFramework.beforeEach(beforeEachFragmentPerTest)} - ${this.testFramework.afterEach(afterEachFragmentPerTest)} - ${cloneFunctionFragment}; - `); - } else { - return null; - } - } - - public retrieveStatementMapsPerFile(): StatementMapDictionary { - const statementMapsPerFile: StatementMapDictionary = Object.create(null); - Object.keys(this.coverageInstrumenterStreamPerFile) - .forEach(key => statementMapsPerFile[key] = this.coverageInstrumenterStreamPerFile[key].statementMap); - return statementMapsPerFile; - } - - private createStreamForFile(coverageVariable: string, fileName: string) { - const stream = new CoverageInstrumenterStream(coverageVariable, fileName); - this.coverageInstrumenterStreamPerFile[fileName] = stream; - return stream; - } -} - -const cloneFunctionFragment = ` - function clone(source) { - var result = source; - if (Array.isArray(source)) { - result = []; - source.forEach(function (child, index) { - result[index] = clone(child); - }); - } else if (typeof source == "object") { - result = {}; - for (var i in source) { - result[i] = clone(source[i]); - } - } - return result; - }`; - -const beforeEachFragmentPerTest = ` -if (!globalCoverage.baseline && window.${COVERAGE_CURRENT_TEST_VARIABLE_NAME}) { - globalCoverage.baseline = clone(window.${COVERAGE_CURRENT_TEST_VARIABLE_NAME}); -}`; - -const afterEachFragmentPerTest = ` - globalCoverage.deviations[id] = coverageResult = {}; - id++; - var coveragePerFile = window.${COVERAGE_CURRENT_TEST_VARIABLE_NAME}; - if(coveragePerFile) { - Object.keys(coveragePerFile).forEach(function (file) { - var coverage = coveragePerFile[file]; - var baseline = globalCoverage.baseline[file]; - var fileResult = { s: {} }; - var touchedFile = false; - for(var i in coverage.s){ - if(coverage.s[i] !== baseline.s[i]){ - fileResult.s[i] = coverage.s[i]; - touchedFile = true; - } - } - if(touchedFile){ - coverageResult[file] = fileResult; - } - }); - }`; \ No newline at end of file diff --git a/packages/stryker/src/coverage/CoverageInstrumenterStream.ts b/packages/stryker/src/coverage/CoverageInstrumenterStream.ts deleted file mode 100644 index 3f5cb53b08..0000000000 --- a/packages/stryker/src/coverage/CoverageInstrumenterStream.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { StatementMap } from 'stryker-api/test_runner'; -import { Transform, TransformOptions } from 'stream'; -import { Instrumenter } from 'istanbul'; -import { getLogger } from 'log4js'; - -const coverageObjRegex = /\{.*"path".*"fnMap".*"statementMap".*"branchMap".*\}/g; - -/** - * Represents a stream responsible to add code coverage instrumentation and reporting back on the statement map - */ -export default class CoverageInstrumenterStream extends Transform { - - private readonly log = getLogger(CoverageInstrumenterStream.name); - - private source: string; - public statementMap: StatementMap; - - constructor(public coverageVariable: string, private filename: string, opts?: TransformOptions) { - super(opts); - this.source = ''; - } - - _transform(chunk: Buffer | string, encoding: string, callback: Function): void { - this.source += chunk.toString(); - callback(); - } - - _flush(callback: Function): void { - try { - const instrumenter = new Instrumenter({ coverageVariable: this.coverageVariable }); - const instrumentedCode = instrumenter.instrumentSync(this.source, this.filename); - coverageObjRegex.lastIndex = 0; - const coverageObjectMatch = coverageObjRegex.exec(instrumentedCode) + ''; - const coverageObj = JSON.parse(coverageObjectMatch); - this.statementMap = coverageObj.statementMap; - Object.keys(this.statementMap).forEach(key => { - // Lines from istanbul are one-based, lines in Stryker are 0-based - this.statementMap[key].end.line--; - this.statementMap[key].start.line--; - }); - this.push(instrumentedCode); - } catch (err) { - const error = `Error while instrumenting file "${this.filename}", error was: ${err.toString()}`; - this.log.error(error); - this.push(this.source); - } - callback(); - } -} \ No newline at end of file diff --git a/packages/stryker/src/process/InitialTestExecutor.ts b/packages/stryker/src/process/InitialTestExecutor.ts index 2b4f7eb1cf..ff9fd1f92c 100644 --- a/packages/stryker/src/process/InitialTestExecutor.ts +++ b/packages/stryker/src/process/InitialTestExecutor.ts @@ -2,13 +2,13 @@ import { EOL } from 'os'; import { RunStatus, RunResult, TestResult, TestStatus } from 'stryker-api/test_runner'; import { TestFramework } from 'stryker-api/test_framework'; import { Config } from 'stryker-api/config'; -import { TranspileResult } from 'stryker-api/transpile'; +import { TranspileResult, TranspilerOptions, Transpiler } from 'stryker-api/transpile'; import { File } from 'stryker-api/core'; import TranspilerFacade from '../transpiler/TranspilerFacade'; -import CoverageInstrumenter from '../coverage/CoverageInstrumenter'; import { getLogger } from 'log4js'; import Sandbox from '../Sandbox'; import Timer from '../utils/Timer'; +import CoverageInstrumenterTranspiler, { StatementMapDictionary } from '../transpiler/CoverageInstrumenterTranspiler'; // The initial run might take a while. // For example: angular-bootstrap takes up to 45 seconds. @@ -18,13 +18,14 @@ const INITIAL_RUN_TIMEOUT = 60 * 1000 * 5; export interface InitialTestRunResult { runResult: RunResult; transpiledFiles: File[]; + statementMaps: StatementMapDictionary; } export default class InitialTestExecutor { private readonly log = getLogger(InitialTestExecutor.name); - constructor(private options: Config, private files: File[], private coverageInstrumenter: CoverageInstrumenter, private testFramework: TestFramework | null, private timer: Timer) { + constructor(private options: Config, private files: File[], private testFramework: TestFramework | null, private timer: Timer) { } async run(): Promise { @@ -39,28 +40,18 @@ export default class InitialTestExecutor { } } - private createDryRunResult(): InitialTestRunResult { - return { - runResult: { - status: RunStatus.Complete, - tests: [], - errorMessages: [] - }, - transpiledFiles: [] - }; - } - - private async initialRunInSandbox(): Promise<{ runResult: RunResult, transpiledFiles: File[] }> { - const transpiler = new TranspilerFacade({ config: this.options, keepSourceMaps: true }); - const transpileResult = transpiler.transpile(this.files); + private async initialRunInSandbox(): Promise { + const coverageInstrumenterTranspiler = this.createCoverageInstrumenterTranspiler(); + const transpilerFacade = this.createTranspilerFacade(coverageInstrumenterTranspiler); + const transpileResult = transpilerFacade.transpile(this.files); if (transpileResult.error) { throw new Error(`Could not transpile input files: ${transpileResult.error}`); } else { this.logTranspileResult(transpileResult); - const sandbox = await Sandbox.create(this.options, 0, transpileResult.outputFiles, this.testFramework, this.coverageInstrumenter); + const sandbox = await Sandbox.create(this.options, 0, transpileResult.outputFiles, this.testFramework); const runResult = await sandbox.run(INITIAL_RUN_TIMEOUT); await sandbox.dispose(); - return { runResult, transpiledFiles: transpileResult.outputFiles }; + return { runResult, transpiledFiles: transpileResult.outputFiles, statementMaps: coverageInstrumenterTranspiler.statementMapsPerFile }; } } @@ -88,6 +79,36 @@ export default class InitialTestExecutor { throw new Error('Something went wrong in the initial test run'); } + private createDryRunResult(): InitialTestRunResult { + return { + runResult: { + status: RunStatus.Complete, + tests: [], + errorMessages: [] + }, + transpiledFiles: [], + statementMaps: Object.create(null) + }; + } + + /** + * Creates a facade for the transpile pipeline. + * Also includes the coverage instrumenter transpiler, + * which is used to instrument for code coverage when needed. + */ + private createTranspilerFacade(coverageInstrumenterTranspiler: CoverageInstrumenterTranspiler): Transpiler { + const transpilerSettings: TranspilerOptions = { config: this.options, keepSourceMaps: true }; + const transpiler = new TranspilerFacade(transpilerSettings, { + name: CoverageInstrumenterTranspiler.name, + transpiler: coverageInstrumenterTranspiler + }); + return transpiler; + } + + private createCoverageInstrumenterTranspiler() { + return new CoverageInstrumenterTranspiler({ keepSourceMaps: true, config: this.options }, this.testFramework); + } + private logTranspileResult(transpileResult: TranspileResult) { if (this.options.transpilers.length && this.log.isDebugEnabled()) { this.log.debug(`Transpiled files in order:${EOL}${transpileResult.outputFiles.map(f => `${f.name} (included: ${f.included})`).join(EOL)}`); diff --git a/packages/stryker/src/process/MutationTestExecutor.ts b/packages/stryker/src/process/MutationTestExecutor.ts index cad347a9fb..1a75269bb7 100644 --- a/packages/stryker/src/process/MutationTestExecutor.ts +++ b/packages/stryker/src/process/MutationTestExecutor.ts @@ -14,13 +14,13 @@ import SandboxPool from '../SandboxPool'; export default class MutationTestExecutor { - constructor(private config: Config, private inputFiles: File[], private transpiledFiles: File[], private testFramework: TestFramework | null, private reporter: StrictReporter) { + constructor(private config: Config, private inputFiles: File[], private testFramework: TestFramework | null, private reporter: StrictReporter) { } async run(allMutants: TestableMutant[]): Promise { - const sandboxPool = new SandboxPool(this.config, this.testFramework, this.transpiledFiles); const mutantTranspiler = new MutantTranspiler(this.config); - await mutantTranspiler.initialize(this.inputFiles); + const transpileResult = await mutantTranspiler.initialize(this.inputFiles); + const sandboxPool = new SandboxPool(this.config, this.testFramework, transpileResult.outputFiles); const result = await this.runInsideSandboxes( sandboxPool.streamSandboxes(), mutantTranspiler.transpileMutants(allMutants)); @@ -61,7 +61,7 @@ function runInSandbox([transpiledMutant, sandbox]: [TranspiledMutant, Sandbox]): if (transpiledMutant.transpileResult.error) { const result = transpiledMutant.mutant.result(MutantStatus.TranspileError, []); return Promise.resolve({ sandbox, result }); - } else if (!transpiledMutant.mutant.scopedTestIds.length) { + } else if (!transpiledMutant.mutant.selectedTests.length) { const result = transpiledMutant.mutant.result(MutantStatus.NoCoverage, []); return Promise.resolve({ sandbox, result }); } else { diff --git a/packages/stryker/src/transpiler/CoverageInstrumenterTranspiler.ts b/packages/stryker/src/transpiler/CoverageInstrumenterTranspiler.ts new file mode 100644 index 0000000000..ff8a5c6a45 --- /dev/null +++ b/packages/stryker/src/transpiler/CoverageInstrumenterTranspiler.ts @@ -0,0 +1,180 @@ +import { Transpiler, TranspileResult, FileLocation, TranspilerOptions } from 'stryker-api/transpile'; +import { File, FileKind, TextFile } from 'stryker-api/core'; +import { StatementMap } from 'stryker-api/test_runner'; +import { Instrumenter } from 'istanbul'; +import { errorToString, wrapInClosure } from '../utils/objectUtils'; +import { TestFramework } from 'stryker-api/test_framework'; +import { Logger, getLogger } from 'log4js'; + +const coverageObjRegex = /\{.*"path".*"fnMap".*"statementMap".*"branchMap".*\}/g; +const COVERAGE_CURRENT_TEST_VARIABLE_NAME = '__strykerCoverageCurrentTest__'; + +export interface StatementMapDictionary { + [file: string]: StatementMap; +} + +export default class CoverageInstrumenterTranspiler implements Transpiler { + + private instrumenter: Instrumenter; + public statementMapsPerFile: StatementMapDictionary = Object.create(null); + private log: Logger; + + constructor(private settings: TranspilerOptions, private testFramework: TestFramework | null) { + this.instrumenter = new Instrumenter({ coverageVariable: this.coverageVariable }); + this.log = getLogger(CoverageInstrumenterTranspiler.name); + } + + transpile(files: File[]): TranspileResult { + try { + const result: TranspileResult = { + outputFiles: files.map(file => this.instrumentFileIfNeeded(file)), + error: null + }; + return this.addCollectCoverageFileIfNeeded(result); + } catch (error) { + return this.errorResult(errorToString(error)); + } + } + + getMappedLocation(sourceFileLocation: FileLocation): FileLocation { + return sourceFileLocation; + } + + /** + * Coverage variable *must* have the name '__coverage__'. Only that variable + * is reported back to the TestRunner process when using one of the karma + * test framework adapters (karma-jasmine, karma-mocha, ...). + * + * However, when coverageAnalysis is 'perTest' we don't choose that variable name right away, + * because we need that variable to hold all coverage results per test. Instead, we use __strykerCoverageCurrentTest__ + * and after each test copy over the value of that current test to the global coverage object __coverage__ + */ + private get coverageVariable() { + switch (this.settings.config.coverageAnalysis) { + case 'perTest': + return COVERAGE_CURRENT_TEST_VARIABLE_NAME; + default: + return '__coverage__'; + } + } + + private retrieveStatementMap(instrumentedCode: string): StatementMap { + coverageObjRegex.lastIndex = 0; + const coverageObjectMatch = coverageObjRegex.exec(instrumentedCode) + ''; + const statementMap = JSON.parse(coverageObjectMatch).statementMap as StatementMap; + Object.keys(statementMap).forEach(key => { + // Lines from istanbul are one-based, lines in Stryker are 0-based + statementMap[key].end.line--; + statementMap[key].start.line--; + }); + return statementMap; + } + + private instrumentFileIfNeeded(file: File) { + if (this.settings.config.coverageAnalysis !== 'off' && file.kind === FileKind.Text && file.mutated) { + return this.instrumentFile(file); + } else { + return file; + } + } + + private instrumentFile(sourceFile: TextFile): TextFile { + try { + const content = this.instrumenter.instrumentSync(sourceFile.content, sourceFile.name); + this.statementMapsPerFile[sourceFile.name] = this.retrieveStatementMap(content); + return { + mutated: sourceFile.mutated, + included: sourceFile.included, + name: sourceFile.name, + transpiled: sourceFile.transpiled, + kind: FileKind.Text, + content + }; + } catch (error) { + throw new Error(`Could not instrument "${sourceFile.name}" for code coverage. ${errorToString(error)}`); + } + } + + private addCollectCoverageFileIfNeeded(result: TranspileResult): TranspileResult { + if (Object.keys(this.statementMapsPerFile).length && this.settings.config.coverageAnalysis === 'perTest') { + if (this.testFramework) { + // Add piece of javascript to collect coverage per test results + const content = this.coveragePerTestFileContent(this.testFramework); + const fileName = '____collectCoveragePerTest____.js'; + result.outputFiles.unshift({ + kind: FileKind.Text, + name: fileName, + included: true, + transpiled: false, + mutated: false, + content + }); + this.log.debug(`Adding test hooks file for coverageAnalysis "perTest": ${fileName}`); + } else { + return this.errorResult('Cannot measure coverage results per test, there is no testFramework and thus no way of executing code right before and after each test.'); + } + } + return result; + } + + private coveragePerTestFileContent(testFramework: TestFramework): string { + return wrapInClosure(` + var id = 0, globalCoverage, coverageResult; + window.__coverage__ = globalCoverage = { deviations: {} }; + ${testFramework.beforeEach(beforeEachFragmentPerTest)} + ${testFramework.afterEach(afterEachFragmentPerTest)} + ${cloneFunctionFragment}; + `); + } + + private errorResult(error: string) { + return { + error, + outputFiles: [] + }; + } +} + +const cloneFunctionFragment = ` +function clone(source) { + var result = source; + if (Array.isArray(source)) { + result = []; + source.forEach(function (child, index) { + result[index] = clone(child); + }); + } else if (typeof source == "object") { + result = {}; + for (var i in source) { + result[i] = clone(source[i]); + } + } + return result; +}`; + +const beforeEachFragmentPerTest = ` +if (!globalCoverage.baseline && window.${COVERAGE_CURRENT_TEST_VARIABLE_NAME}) { +globalCoverage.baseline = clone(window.${COVERAGE_CURRENT_TEST_VARIABLE_NAME}); +}`; + +const afterEachFragmentPerTest = ` +globalCoverage.deviations[id] = coverageResult = {}; +id++; +var coveragePerFile = window.${COVERAGE_CURRENT_TEST_VARIABLE_NAME}; +if(coveragePerFile) { +Object.keys(coveragePerFile).forEach(function (file) { + var coverage = coveragePerFile[file]; + var baseline = globalCoverage.baseline[file]; + var fileResult = { s: {} }; + var touchedFile = false; + for(var i in coverage.s){ + if(coverage.s[i] !== baseline.s[i]){ + fileResult.s[i] = coverage.s[i]; + touchedFile = true; + } + } + if(touchedFile){ + coverageResult[file] = fileResult; + } +}); +}`; \ No newline at end of file diff --git a/packages/stryker/src/transpiler/TranspilerFacade.ts b/packages/stryker/src/transpiler/TranspilerFacade.ts index b2e0d5e7dd..7f632ede03 100644 --- a/packages/stryker/src/transpiler/TranspilerFacade.ts +++ b/packages/stryker/src/transpiler/TranspilerFacade.ts @@ -9,9 +9,12 @@ export default class TranspilerFacade implements Transpiler { private innerTranspilers: NamedTranspiler[]; - constructor(options: TranspilerOptions) { + constructor(options: TranspilerOptions, additionalTranspiler?: { name: string, transpiler: Transpiler }) { this.innerTranspilers = options.config.transpilers .map(transpilerName => new NamedTranspiler(transpilerName, TranspilerFactory.instance().create(transpilerName, options))); + if (additionalTranspiler) { + this.innerTranspilers.push(new NamedTranspiler(additionalTranspiler.name, additionalTranspiler.transpiler)); + } } transpile(files: File[]): TranspileResult { diff --git a/packages/stryker/src/utils/fileUtils.ts b/packages/stryker/src/utils/fileUtils.ts index 9553273abc..755ecf88b9 100644 --- a/packages/stryker/src/utils/fileUtils.ts +++ b/packages/stryker/src/utils/fileUtils.ts @@ -55,30 +55,14 @@ function isBinaryFile(name: string): boolean { * @param data The content of the file. * @returns A promise to eventually save the file. */ -export function writeFile(fileName: string, data: string | Buffer, instrumenter: NodeJS.ReadWriteStream | null = null): Promise { +export function writeFile(fileName: string, data: string | Buffer): Promise { if (Buffer.isBuffer(data)) { return fs.writeFile(fileName, data); - } else if (instrumenter) { - instrumenter.pipe(fs.createWriteStream(fileName, 'utf8')); - return writeToStream(data, instrumenter); } else { return fs.writeFile(fileName, data, 'utf8'); } } -function writeToStream(data: string | Buffer, stream: NodeJS.WritableStream): Promise { - return new Promise((res, rej) => { - stream.end(data as string, (err: any) => { - if (err) { - rej(err); - } else { - res(); - } - }); - }); -} - - export function determineFileKind(fileName: string): FileKind { if (isOnlineFile(fileName)) { return FileKind.Web; diff --git a/packages/stryker/test/helpers/producers.ts b/packages/stryker/test/helpers/producers.ts index ca75a599fb..4df36bfb59 100644 --- a/packages/stryker/test/helpers/producers.ts +++ b/packages/stryker/test/helpers/producers.ts @@ -3,7 +3,7 @@ import { Mutant } from 'stryker-api/mutant'; import { FileLocation, TranspileResult } from 'stryker-api/transpile'; import { Config } from 'stryker-api/config'; import * as sinon from 'sinon'; -import { TestFramework } from 'stryker-api/test_framework'; +import { TestFramework, TestSelection } from 'stryker-api/test_framework'; import { MutantStatus, MatchedMutant, MutantResult, Reporter, ScoreResult } from 'stryker-api/report'; import { MutationScoreThresholds, File, Location, TextFile, BinaryFile, FileKind, WebFile, FileDescriptor } from 'stryker-api/core'; import TestableMutant from '../../src/TestableMutant'; @@ -114,7 +114,7 @@ export const fileLocation = factory({ export const testFramework = factory({ beforeEach(codeFragment: string) { return `beforeEach(){ ${codeFragment}}`; }, afterEach(codeFragment: string) { return `afterEach(){ ${codeFragment}}`; }, - filter(ids: number[]) { return `filter: ${ids}`; } + filter(selections: TestSelection[]) { return `filter: ${selections}`; } }); export const scoreResult = factoryMethod(() => ({ diff --git a/packages/stryker/test/helpers/streamHelpers.ts b/packages/stryker/test/helpers/streamHelpers.ts deleted file mode 100644 index 45ccaf979f..0000000000 --- a/packages/stryker/test/helpers/streamHelpers.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Readable, Stream } from 'stream'; - -export function streamToString(stream: Stream) { - return new Promise(resolve => { - const chunks: string[] = []; - stream.on('data', (chunk: Buffer | string) => { - chunks.push(chunk.toString()); - }); - stream.on('end', () => { - resolve(chunks.join('')); - }); - }); -} - -export function readable(): Readable { - const input = new SimpleReadable(); - return input; -} - -class SimpleReadable extends Readable { - _read() { - // noop - } -} \ No newline at end of file diff --git a/packages/stryker/test/unit/MutantTestMatcherSpec.ts b/packages/stryker/test/unit/MutantTestMatcherSpec.ts index 2e4868dec0..621d1ee67c 100644 --- a/packages/stryker/test/unit/MutantTestMatcherSpec.ts +++ b/packages/stryker/test/unit/MutantTestMatcherSpec.ts @@ -5,16 +5,16 @@ import { expect } from 'chai'; import { RunResult, TestResult, RunStatus, TestStatus, CoverageCollection, CoveragePerTestResult } from 'stryker-api/test_runner'; import { StrykerOptions, File } from 'stryker-api/core'; import { MatchedMutant } from 'stryker-api/report'; -import { StatementMapDictionary } from '../../src/coverage/CoverageInstrumenter'; import MutantTestMatcher from '../../src/MutantTestMatcher'; import currentLogMock from '../helpers/log4jsMock'; import { file, mutant, Mock, mock } from '../helpers/producers'; import TestableMutant from '../../src/TestableMutant'; import SourceFile from '../../src/SourceFile'; import BroadcastReporter from '../../src/reporters/BroadcastReporter'; +import { StatementMapDictionary } from '../../src/transpiler/CoverageInstrumenterTranspiler'; describe('MutantTestMatcher', () => { - + let log: Mock; let sut: MutantTestMatcher; let mutants: Mutant[]; @@ -87,22 +87,24 @@ describe('MutantTestMatcher', () => { it('should add both tests to the mutants', () => { const result = sut.matchWithMutants(); - expect(result[0].scopedTestIds).deep.eq([0, 1]); - expect(result[1].scopedTestIds).deep.eq([0, 1]); + const expectedTestSelection = [{ id: 0, name: 'test one' }, { id: 1, name: 'test two' }]; + expect(result[0].selectedTests).deep.eq(expectedTestSelection); + expect(result[1].selectedTests).deep.eq(expectedTestSelection); }); + it('should have both mutants matched', () => { const result = sut.matchWithMutants(); - let matchedMutants: MatchedMutant[] = [ + const matchedMutants: MatchedMutant[] = [ { mutatorName: result[0].mutatorName, - scopedTestIds: result[0].scopedTestIds, + scopedTestIds: result[0].selectedTests.map(test => test.id), timeSpentScopedTests: result[0].timeSpentScopedTests, fileName: result[0].fileName, replacement: result[0].replacement }, { mutatorName: result[1].mutatorName, - scopedTestIds: result[1].scopedTestIds, + scopedTestIds: result[1].selectedTests.map(test => test.id), timeSpentScopedTests: result[1].timeSpentScopedTests, fileName: result[1].fileName, replacement: result[1].replacement @@ -167,8 +169,8 @@ describe('MutantTestMatcher', () => { it('should not have added the run results to the mutants', () => { const result = sut.matchWithMutants(); - expect(result[0].scopedTestIds).lengthOf(0); - expect(result[1].scopedTestIds).lengthOf(0); + expect(result[0].selectedTests).lengthOf(0); + expect(result[1].selectedTests).lengthOf(0); }); }); @@ -198,8 +200,9 @@ describe('MutantTestMatcher', () => { it('should have added the run results to the mutants', () => { const result = sut.matchWithMutants(); - expect(result[0].scopedTestIds).deep.eq([0, 1]); - expect(result[1].scopedTestIds).deep.eq([0, 1]); + const expectedTestSelection = [{ id: 0, name: 'test one' }, { id: 1, name: 'test two' }]; + expect(result[0].selectedTests).deep.eq(expectedTestSelection); + expect(result[1].selectedTests).deep.eq(expectedTestSelection); }); }); @@ -222,8 +225,9 @@ describe('MutantTestMatcher', () => { it('should add all test results to the mutant that is covered by the baseline', () => { const result = sut.matchWithMutants(); - expect(result[0].scopedTestIds).deep.eq([0, 1]); - expect(result[1].scopedTestIds).deep.eq([0, 1]); + const expectedTestSelection = [{ id: 0, name: 'test one' }, { id: 1, name: 'test two' }]; + expect(result[0].selectedTests).deep.eq(expectedTestSelection); + expect(result[1].selectedTests).deep.eq(expectedTestSelection); }); }); }); @@ -247,7 +251,10 @@ describe('MutantTestMatcher', () => { timeSpentMs: 5 }); sut.enrichWithCoveredTests(testableMutant); - expect(testableMutant.scopedTestIds).deep.eq([0]); + expect(testableMutant.selectedTests).deep.eq([{ + id: 0, + name: 'controllers SearchResultController should open a modal dialog with product details' + }]); }); }); }); @@ -261,8 +268,9 @@ describe('MutantTestMatcher', () => { mutants.push(mutant({ fileName: 'fileWithMutantOne' }), mutant({ fileName: 'fileWithMutantTwo' })); runResult.tests.push(testResult(), testResult()); const result = sut.matchWithMutants(); - expect(result[0].scopedTestIds).deep.eq([0, 1]); - expect(result[1].scopedTestIds).deep.eq([0, 1]); + const expectedTestSelection = [{ id: 0, name: 'name' }, { id: 1, name: 'name' }]; + expect(result[0].selectedTests).deep.eq(expectedTestSelection); + expect(result[1].selectedTests).deep.eq(expectedTestSelection); expect(log.warn).to.have.been.calledWith('No coverage result found, even though coverageAnalysis is "%s". Assuming that all tests cover each mutant. This might have a big impact on the performance.', 'all'); }); }); @@ -275,8 +283,9 @@ describe('MutantTestMatcher', () => { mutants.push(mutant({ fileName: 'fileWithMutantOne' }), mutant({ fileName: 'fileWithMutantTwo' })); runResult.tests.push(testResult(), testResult()); const result = sut.matchWithMutants(); - expect(result[0].scopedTestIds).deep.eq([0, 1]); - expect(result[1].scopedTestIds).deep.eq([0, 1]); + const expectedTestSelection = [{ id: 0, name: 'name' }, { id: 1, name: 'name' }]; + expect(result[0].selectedTests).deep.eq(expectedTestSelection); + expect(result[1].selectedTests).deep.eq(expectedTestSelection); }); }); }); \ No newline at end of file diff --git a/packages/stryker/test/unit/SandboxPoolSpec.ts b/packages/stryker/test/unit/SandboxPoolSpec.ts index bb656b4228..73f9a33b02 100644 --- a/packages/stryker/test/unit/SandboxPoolSpec.ts +++ b/packages/stryker/test/unit/SandboxPoolSpec.ts @@ -41,7 +41,7 @@ describe('SandboxPool', () => { options.maxConcurrentTestRunners = 1; await sut.streamSandboxes().toArray().toPromise(); expect(Sandbox.create).to.have.callCount(1); - expect(Sandbox.create).calledWith(options, 0, expectedInputFiles, expectedTestFramework, null); + expect(Sandbox.create).calledWith(options, 0, expectedInputFiles, expectedTestFramework); }); it('should use cpuCount when maxConcurrentTestRunners is set too high', async () => { @@ -50,7 +50,7 @@ describe('SandboxPool', () => { const actual = await sut.streamSandboxes().toArray().toPromise(); expect(actual).lengthOf(3); expect(Sandbox.create).to.have.callCount(3); - expect(Sandbox.create).calledWith(options, 0, expectedInputFiles, expectedTestFramework, null); + expect(Sandbox.create).calledWith(options, 0, expectedInputFiles, expectedTestFramework); }); it('should use the cpuCount when maxConcurrentTestRunners is <= 0', async () => { @@ -59,7 +59,7 @@ describe('SandboxPool', () => { const actual = await sut.streamSandboxes().toArray().toPromise(); expect(Sandbox.create).to.have.callCount(3); expect(actual).lengthOf(3); - expect(Sandbox.create).calledWith(options, 0, expectedInputFiles, expectedTestFramework, null); + expect(Sandbox.create).calledWith(options, 0, expectedInputFiles, expectedTestFramework); }); it('should use the cpuCount - 1 when a transpiler is configured', async () => { diff --git a/packages/stryker/test/unit/SandboxSpec.ts b/packages/stryker/test/unit/SandboxSpec.ts index 08a784cfb9..41cabf0056 100644 --- a/packages/stryker/test/unit/SandboxSpec.ts +++ b/packages/stryker/test/unit/SandboxSpec.ts @@ -58,75 +58,34 @@ describe('Sandbox', () => { sandbox.stub(ResilientTestRunnerFactory, 'create').returns(testRunner); }); - describe('when constructed with a CoverageInstrumenter', () => { - - let coverageInstrumenter: { - instrumenterStreamForFile: sinon.SinonStub; - hooksForTestRun: sinon.SinonStub; - }; - let expectedInstrumenterStream: any; - - beforeEach(() => { - expectedInstrumenterStream = 'an instrumenter stream'; - coverageInstrumenter = { - instrumenterStreamForFile: sinon.stub(), - hooksForTestRun: sinon.stub().returns('jsm.filter()') - }; - coverageInstrumenter.instrumenterStreamForFile.returns(expectedInstrumenterStream); - }); - - describe('when create()', () => { - - beforeEach(async () => { - sut = await Sandbox.create(options, 3, files, testFrameworkStub, coverageInstrumenter); - }); - - it('should have instrumented the input files', () => { - expect(coverageInstrumenter.instrumenterStreamForFile).calledWith(expectedFileToMutate); - expect(coverageInstrumenter.instrumenterStreamForFile).calledWith(expectedFileToMutate); - expect(fileUtils.writeFile).calledWith(expectedTargetFileToMutate, expectedFileToMutate.content, expectedInstrumenterStream); - }); - - it('should have created the isolated test runner inc framework hook', () => { - const expectedSettings: IsolatedRunnerOptions = { - files: [ - fileDescriptor({ name: expectedTestFrameworkHooksFile, mutated: false, included: true, transpiled: false }), - fileDescriptor({ name: expectedTargetFileToMutate, mutated: true, included: true }), - fileDescriptor({ name: path.join(workingFolder, 'file2'), mutated: false, included: false }), - fileDescriptor({ name: webFileUrl, mutated: false, included: true, kind: FileKind.Web, transpiled: false }) - ], - port: 46, - strykerOptions: options, - sandboxWorkingFolder: workingFolder - }; - expect(ResilientTestRunnerFactory.create).calledWith(options.testRunner, expectedSettings); - }); + it('should copy input files when created', async () => { + sut = await Sandbox.create(options, 3, files, null); + expect(fileUtils.writeFile).calledWith(expectedTargetFileToMutate, textFiles[0].content); + expect(fileUtils.writeFile).calledWith(path.join(workingFolder, 'file2'), textFiles[1].content); + }); + it('should copy a local file when created', async () => { + sut = await Sandbox.create(options, 3, [textFile({ name: 'localFile.js', content: 'foobar' })], null); + expect(fileUtils.writeFile).calledWith(path.join(workingFolder, 'localFile.js'), 'foobar'); + }); - it('should not have written online files', () => { - let expectedBaseFolder = webFileUrl.substr(workingFolder.length - 1); // The Sandbox expects all files to be absolute paths. An online file is not an absolute path. + describe('when constructed with a testFramework', () => { - expect(mkdirp.sync).not.calledWith(workingFolder + path.dirname(expectedBaseFolder)); - expect(fileUtils.writeFile).not.calledWith(webFileUrl, sinon.match.any, sinon.match.any); - }); + beforeEach(async () => { + sut = await Sandbox.create(options, 3, files, testFrameworkStub); }); - }); - describe('when constructed with a testFramework but without a CoverageInstrumenter', () => { + it('should not have written online files', () => { + let expectedBaseFolder = webFileUrl.substr(workingFolder.length - 1); // The Sandbox expects all files to be absolute paths. An online file is not an absolute path. - beforeEach(async () => { - sut = await Sandbox.create(options, 3, files, testFrameworkStub, null); + expect(mkdirp.sync).not.calledWith(workingFolder + path.dirname(expectedBaseFolder)); + expect(fileUtils.writeFile).not.calledWith(webFileUrl, sinon.match.any, sinon.match.any); }); it('should have created a workingFolder', () => { expect(TempFolder.instance().createRandomFolder).to.have.been.calledWith('sandbox'); }); - it('should have copied the input files', () => { - expect(fileUtils.writeFile).calledWith(expectedTargetFileToMutate, textFiles[0].content); - expect(fileUtils.writeFile).calledWith(path.join(workingFolder, 'file2'), textFiles[1].content); - }); - it('should have created the isolated test runner without framework hook', () => { const expectedSettings: IsolatedRunnerOptions = { files: [ @@ -180,7 +139,7 @@ describe('Sandbox', () => { }); it('should filter the scoped tests', () => { - expect(testFrameworkStub.filter).to.have.been.calledWith(transpiledMutant.mutant.scopedTestIds); + expect(testFrameworkStub.filter).to.have.been.calledWith(transpiledMutant.mutant.selectedTests); }); it('should write the filter code fragment to hooks file', () => { @@ -201,9 +160,9 @@ describe('Sandbox', () => { }); }); - describe('when constructed without a testFramework or CoverageInstrumenter', () => { + describe('when constructed without a testFramework', () => { beforeEach(async () => { - sut = await Sandbox.create(options, 3, files, null, null); + sut = await Sandbox.create(options, 3, files, null); }); diff --git a/packages/stryker/test/unit/TestableMutantSpec.ts b/packages/stryker/test/unit/TestableMutantSpec.ts index c00102468c..919ae951e9 100644 --- a/packages/stryker/test/unit/TestableMutantSpec.ts +++ b/packages/stryker/test/unit/TestableMutantSpec.ts @@ -26,9 +26,9 @@ describe('TestableMutant', () => { }); it('should reflect timeSpentScopedTests and scopedTestIds', () => { - sut.addAllTestResults(runResult({ tests: [testResult({ timeSpentMs: 12 }), testResult({ timeSpentMs: 42 })] })); + sut.addAllTestResults(runResult({ tests: [testResult({ name: 'spec1', timeSpentMs: 12 }), testResult({ name: 'spec2', timeSpentMs: 42 })] })); expect(sut.timeSpentScopedTests).eq(54); - expect(sut.scopedTestIds).deep.eq([0, 1]); + expect(sut.selectedTests).deep.eq([{ id: 0, name: 'spec1' }, { id: 1, name: 'spec2' }]); }); it('should calculate position using sourceFile', () => { diff --git a/packages/stryker/test/unit/coverage/CoverageInstrumenterSpec.ts b/packages/stryker/test/unit/coverage/CoverageInstrumenterSpec.ts deleted file mode 100644 index 1c735c58ae..0000000000 --- a/packages/stryker/test/unit/coverage/CoverageInstrumenterSpec.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { expect } from 'chai'; -import { PassThrough } from 'stream'; -import { readable, streamToString } from '../../helpers/streamHelpers'; -import { TestFramework } from 'stryker-api/test_framework'; -import { file } from '../../helpers/producers'; -import CoverageInstrumenter from '../../../src/coverage/CoverageInstrumenter'; -import CoverageInstrumenterStream from '../../../src/coverage/CoverageInstrumenterStream'; - -describe('CoverageInstrumenter', () => { - let sut: CoverageInstrumenter; - let testFramework: TestFramework; - const streamToFile = (name: string, mutated: boolean, content: string): Promise => { - const stream = sut.instrumenterStreamForFile(file({ name, mutated, included: true })); - const input = readable(); - input.push(content); - input.push(null); - return streamToString(input.pipe(stream)); - }; - - beforeEach(() => { - testFramework = { - beforeEach: function (codeFragment: string): string { - return `beforeEach() { ${codeFragment} }`; - }, - afterEach: function (codeFragment: string): string { - return `afterEach() { ${codeFragment} }`; - }, - filter: function (testIds: number[]): string { - return `filter(${JSON.stringify(testIds)})`; - } - }; - }); - - describe('with coverageAnalysis "perTest"', () => { - - beforeEach(() => { - sut = new CoverageInstrumenter('perTest', testFramework); - }); - - describe('when hooksForTestRun()', () => { - it('should return the perTest hooks', () => { - const actual = sut.hooksForTestRun(); - expect(actual).to.have.length.greaterThan(30); - expect(actual).to.contain('beforeEach()'); - expect(actual).to.contain('afterEach()'); - }); - }); - - describe('when instrumenterStreamForFile()', () => { - - it('should retrieve an intrumenter stream for mutated files', () => { - const actual = sut.instrumenterStreamForFile(file({ name: '', mutated: true, included: true })); - expect(actual).to.be.an.instanceof(CoverageInstrumenterStream); - if (actual instanceof CoverageInstrumenterStream) { - expect(actual.coverageVariable).to.be.eq('__strykerCoverageCurrentTest__'); - } - }); - - it('should retrieve a PassThrough stream for non-mutated files', () => - expect(sut.instrumenterStreamForFile(file({ name: '', mutated: false, included: true }))).to.be.an.instanceof(PassThrough)); - }); - }); - - describe('with coverageAnalysis "off"', () => { - - beforeEach(() => { - sut = new CoverageInstrumenter('off', testFramework); - }); - - describe('when hooksForTestRun()', () => { - it('should return null', () => { - expect(sut.hooksForTestRun()).be.null; - }); - }); - - describe('when instrumenterStreamForFile()', () => { - it('should retrieve a PassThrough stream for mutated files', () => - expect(sut.instrumenterStreamForFile(file({ name: '', mutated: true, included: true }))).to.be.an.instanceof(PassThrough)); - }); - - describe('retrieveStatementMapsPerFile() with 2 streams', () => { - - beforeEach(() => Promise.all([ - streamToFile('1', true, 'function(){}'), - streamToFile('3', false, 'function(){}') - ])); - - it('should retrieve 0 statement maps', () => { - const actual = sut.retrieveStatementMapsPerFile(); - expect(Object.keys(actual)).to.have.lengthOf(0); - }); - }); - }); - - describe('with coverageAnalysis "all"', () => { - - beforeEach(() => { - sut = new CoverageInstrumenter('all', testFramework); - }); - - describe('when hooksForTestRun()', () => { - it('should return null', () => { - expect(sut.hooksForTestRun()).null; - }); - }); - - describe('when instrumenterStreamForFile()', () => { - it('should retrieve a CoverageInstrumenterStream stream for mutated files', () => { - const actual = sut.instrumenterStreamForFile(file({ name: '', mutated: true, included: true })); - expect(actual).to.be.an.instanceof(CoverageInstrumenterStream); - if (actual instanceof CoverageInstrumenterStream) { - expect(actual.coverageVariable).to.be.eq('__coverage__'); - } - }); - }); - - describe('retrieveStatementMapsPerFile() with 2 streams', () => { - - beforeEach(() => Promise.all([ - streamToFile('1', true, 'function a (){}'), - streamToFile('2', true, 'function b (){}'), - streamToFile('3', false, 'function c (){}') - ])); - - it('should retrieve 2 statement maps', () => { - const actual = sut.retrieveStatementMapsPerFile(); - expect(Object.keys(actual)).to.have.lengthOf(2); - expect(actual['1']).to.be.ok; - expect(actual['2']).to.be.ok; - }); - }); - }); -}); \ No newline at end of file diff --git a/packages/stryker/test/unit/coverage/CoverageInstrumenterStreamSpec.ts b/packages/stryker/test/unit/coverage/CoverageInstrumenterStreamSpec.ts deleted file mode 100644 index d37144f05e..0000000000 --- a/packages/stryker/test/unit/coverage/CoverageInstrumenterStreamSpec.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { expect } from 'chai'; -import { Logger } from 'log4js'; -import { Readable, Duplex } from 'stream'; -import { FileDescriptor } from 'stryker-api/core'; -import CoverageInstrumenterStream from '../../../src/coverage/CoverageInstrumenterStream'; -import currentLogMock from '../../helpers/log4jsMock'; -import { streamToString, readable } from '../../helpers/streamHelpers'; -import { StatementMap } from 'stryker-api/test_runner'; -import { Mock } from '../../helpers/producers'; - -describe('CoverageInstrumenterStream', () => { - let log: Mock; - let sut: CoverageInstrumenterStream; - let inputFiles: FileDescriptor[]; - const filename = 'thefile.js'; - - beforeEach(() => { - log = currentLogMock(); - inputFiles = []; - sut = new CoverageInstrumenterStream('myCoverageVariable', filename); - }); - - it('should extend Duplex', () => expect(sut).to.be.instanceOf(Duplex)); - - describe('when piped', () => { - let input: Readable; - let output: Promise; - - beforeEach(() => { - input = readable(); - output = streamToString(input.pipe(sut)); - }); - - describe('when input is a valid javascript file', () => { - let statementMap: StatementMap; - - beforeEach(() => { - input.push('function something () {', 'utf8'); - input.push('}', 'utf8'); - input.push(null); // signal the end - statementMap = { '1': { start: { line: 0, column: 0 }, end: { line: 0, column: 24 } } }; - }); - - it('should instrument the input', () => - expect(output).to.eventually.contain('function something(){__cov_').and.contain('.f[\'1\']++')); - - it('should contain the statement map', () => output.then(() => { - expect(sut.statementMap).to.deep.eq(statementMap); - })); - }); - - describe('when input is invalid javascript', () => { - const expected = 'function something {}'; - - beforeEach(() => { - input.push(expected); - input.push(null); - return output; - }); - - it('should just pass through the input', () => expect(output).to.eventually.eq(expected)); - - it('should log the error', () => expect(log.error).to.have.been.calledWith('Error while instrumenting file "thefile.js", error was: Error: Line 1: Unexpected token {')); - }); - }); -}); diff --git a/packages/stryker/test/unit/process/InitialTestExecutorSpec.ts b/packages/stryker/test/unit/process/InitialTestExecutorSpec.ts index a71b91e170..432732ad8e 100644 --- a/packages/stryker/test/unit/process/InitialTestExecutorSpec.ts +++ b/packages/stryker/test/unit/process/InitialTestExecutorSpec.ts @@ -7,9 +7,9 @@ import { File } from 'stryker-api/core'; import { Config } from 'stryker-api/config'; import * as producers from '../../helpers/producers'; import { TestFramework } from 'stryker-api/test_framework'; -import CoverageInstrumenter from '../../../src/coverage/CoverageInstrumenter'; +import CoverageInstrumenterTranspiler, * as coverageInstrumenterTranspiler from '../../../src/transpiler/CoverageInstrumenterTranspiler'; import TranspilerFacade, * as transpilerFacade from '../../../src/transpiler/TranspilerFacade'; -import { TranspileResult } from 'stryker-api/transpile'; +import { TranspileResult, TranspilerOptions } from 'stryker-api/transpile'; import { RunStatus, RunResult, TestStatus } from 'stryker-api/test_runner'; import currentLogMock from '../../helpers/log4jsMock'; import Timer from '../../../src/utils/Timer'; @@ -21,7 +21,7 @@ describe('InitialTestExecutor run', () => { let strykerSandboxMock: producers.Mock; let sut: InitialTestExecutor; let testFrameworkMock: TestFramework; - let coverageInstrumenter: CoverageInstrumenter; + let coverageInstrumenterTranspilerMock: producers.Mock; let options: Config; let transpilerFacadeMock: producers.Mock; let transpileResultMock: TranspileResult; @@ -32,10 +32,11 @@ describe('InitialTestExecutor run', () => { log = currentLogMock(); strykerSandboxMock = producers.mock(StrykerSandbox); transpilerFacadeMock = producers.mock(TranspilerFacade); + coverageInstrumenterTranspilerMock = producers.mock(CoverageInstrumenterTranspiler); sandbox.stub(StrykerSandbox, 'create').resolves(strykerSandboxMock); sandbox.stub(transpilerFacade, 'default').returns(transpilerFacadeMock); + sandbox.stub(coverageInstrumenterTranspiler, 'default').returns(coverageInstrumenterTranspilerMock); testFrameworkMock = producers.testFramework(); - coverageInstrumenter = new CoverageInstrumenter('off', testFrameworkMock); transpileResultMock = producers.transpileResult({ outputFiles: [ producers.textFile({ name: 'transpiled-file-1.js' }), @@ -51,7 +52,7 @@ describe('InitialTestExecutor run', () => { describe('without input files', () => { it('should log a warning and cancel the test run', async () => { - sut = new InitialTestExecutor(options, [], coverageInstrumenter, testFrameworkMock, timer as any); + sut = new InitialTestExecutor(options, [], testFrameworkMock, timer as any); const result = await sut.run(); expect(result.runResult.status).to.be.eq(RunStatus.Complete); expect(log.info).to.have.been.calledWith('No files have been found. Aborting initial test run.'); @@ -64,12 +65,12 @@ describe('InitialTestExecutor run', () => { beforeEach(() => { files = [producers.textFile({ name: '', mutated: true, included: true, content: '' })]; - sut = new InitialTestExecutor(options, files, coverageInstrumenter, testFrameworkMock, timer as any); + sut = new InitialTestExecutor(options, files, testFrameworkMock, timer as any); }); it('should create a sandbox with correct arguments', async () => { await sut.run(); - expect(StrykerSandbox.create).calledWith(options, 0, transpileResultMock.outputFiles, testFrameworkMock, coverageInstrumenter); + expect(StrykerSandbox.create).calledWith(options, 0, transpileResultMock.outputFiles, testFrameworkMock); }); it('should initialize, run and dispose the sandbox', async () => { @@ -79,9 +80,13 @@ describe('InitialTestExecutor run', () => { }); it('should pass through the result', async () => { + coverageInstrumenterTranspilerMock.statementMapsPerFile = { someFile: {} } as any; const expectedResult: InitialTestRunResult = { runResult: expectedRunResult, - transpiledFiles: transpileResultMock.outputFiles + transpiledFiles: transpileResultMock.outputFiles, + statementMaps: { + someFile: {} + } }; const actualRunResult = await sut.run(); expect(actualRunResult).deep.eq(expectedResult); @@ -117,7 +122,6 @@ describe('InitialTestExecutor run', () => { expect(log.info).to.have.been.calledWith('Initial test run succeeded. Ran %s tests in %s.', 2); }); - it('should log when there were no tests', async () => { while (expectedRunResult.tests.pop()); await sut.run(); @@ -130,6 +134,16 @@ describe('InitialTestExecutor run', () => { await expect(sut.run()).rejectedWith(expectedError); }); + it('should add the coverage instrumenter transpiler', async () => { + await sut.run(); + const expectedSettings: TranspilerOptions = { + config: options, + keepSourceMaps: true + }; + expect(coverageInstrumenterTranspiler.default).calledWithNew; + expect(coverageInstrumenterTranspiler.default).calledWith(expectedSettings, testFrameworkMock); + }); + describe('and run has test failures', () => { beforeEach(() => { diff --git a/packages/stryker/test/unit/process/MutationTestExecutorSpec.ts b/packages/stryker/test/unit/process/MutationTestExecutorSpec.ts index 56d9631ed0..f1507eec38 100644 --- a/packages/stryker/test/unit/process/MutationTestExecutorSpec.ts +++ b/packages/stryker/test/unit/process/MutationTestExecutorSpec.ts @@ -5,6 +5,7 @@ import { Config } from 'stryker-api/config'; import { File } from 'stryker-api/core'; import { TestFramework } from 'stryker-api/test_framework'; import { RunStatus, TestStatus } from 'stryker-api/test_runner'; +import { TranspileResult } from 'stryker-api/transpile'; import Sandbox from '../../../src/Sandbox'; import BroadcastReporter from '../../../src/reporters/BroadcastReporter'; import MutantTestExecutor from '../../../src/process/MutationTestExecutor'; @@ -12,7 +13,7 @@ import TranspiledMutant from '../../../src/TranspiledMutant'; import { MutantStatus } from 'stryker-api/report'; import MutantTranspiler, * as mutantTranspiler from '../../../src/transpiler/MutantTranspiler'; import SandboxPool, * as sandboxPool from '../../../src/SandboxPool'; -import { transpiledMutant, testResult, Mock, mock, textFile, config, testFramework, testableMutant, mutantResult } from '../../helpers/producers'; +import { transpiledMutant, testResult, Mock, mock, textFile, config, testFramework, testableMutant, mutantResult, transpileResult } from '../../helpers/producers'; import '../../helpers/globals'; import TestableMutant from '../../../src/TestableMutant'; @@ -33,23 +34,23 @@ describe('MutationTestExecutor', () => { let testFrameworkMock: TestFramework; let transpiledMutants: TranspiledMutant[]; let inputFiles: File[]; - let transpiledFiles: File[]; let reporter: Mock; let expectedConfig: Config; let sut: MutantTestExecutor; let mutants: TestableMutant[]; + let initialTranspileResult: TranspileResult; beforeEach(() => { sandboxPoolMock = mock(SandboxPool); mutantTranspilerMock = mock(MutantTranspiler); - mutantTranspilerMock.initialize.resolves(); + initialTranspileResult = transpileResult({ outputFiles: [textFile(), textFile()] }); + mutantTranspilerMock.initialize.resolves(initialTranspileResult); sandboxPoolMock.disposeAll.resolves(); testFrameworkMock = testFramework(); sandbox.stub(sandboxPool, 'default').returns(sandboxPoolMock); sandbox.stub(mutantTranspiler, 'default').returns(mutantTranspilerMock); reporter = mock(BroadcastReporter); inputFiles = [textFile({ name: 'input.ts' })]; - transpiledFiles = [textFile({ name: 'output.js' })]; expectedConfig = config(); mutants = [testableMutant()]; }); @@ -57,7 +58,7 @@ describe('MutationTestExecutor', () => { describe('run', () => { beforeEach(async () => { - sut = new MutantTestExecutor(expectedConfig, inputFiles, transpiledFiles, testFrameworkMock, reporter); + sut = new MutantTestExecutor(expectedConfig, inputFiles, testFrameworkMock, reporter); const sandbox = mock(Sandbox); sandbox.runMutant.resolves(mutantResult()); sandboxPoolMock.streamSandboxes.returns(Observable.of(sandbox)); @@ -70,7 +71,7 @@ describe('MutationTestExecutor', () => { expect(mutantTranspiler.default).calledWithNew; }); it('should create the sandbox pool', () => { - expect(sandboxPool.default).calledWith(expectedConfig, testFrameworkMock, transpiledFiles); + expect(sandboxPool.default).calledWith(expectedConfig, testFrameworkMock, initialTranspileResult.outputFiles); expect(sandboxPool.default).calledWithNew; }); @@ -96,7 +97,7 @@ describe('MutationTestExecutor', () => { mutantTranspilerMock.transpileMutants.returns(Observable.of(...transpiledMutants)); sandboxPoolMock.streamSandboxes.returns(Observable.of(...[firstSandbox, secondSandbox])); - sut = new MutantTestExecutor(config(), inputFiles, transpiledFiles, testFrameworkMock, reporter); + sut = new MutantTestExecutor(config(), inputFiles, testFrameworkMock, reporter); // The uncovered and transpile errors should not be run in a sandbox // Mock first sandbox to return first success, then failed diff --git a/packages/stryker/test/unit/transpiler/CoverageInstrumenterTranspilerSpec.ts b/packages/stryker/test/unit/transpiler/CoverageInstrumenterTranspilerSpec.ts new file mode 100644 index 0000000000..5c722729a9 --- /dev/null +++ b/packages/stryker/test/unit/transpiler/CoverageInstrumenterTranspilerSpec.ts @@ -0,0 +1,99 @@ +import { expect } from 'chai'; +import { Config } from 'stryker-api/config'; +import { TextFile } from 'stryker-api/core'; +import CoverageInstrumenterTranspiler from '../../../src/transpiler/CoverageInstrumenterTranspiler'; +import { textFile, binaryFile, webFile, testFramework } from '../../helpers/producers'; + +describe('CoverageInstrumenterTranspiler', () => { + let sut: CoverageInstrumenterTranspiler; + let config: Config; + + beforeEach(() => { + config = new Config(); + }); + + it('should not instrument any code when coverage analysis is off', () => { + sut = new CoverageInstrumenterTranspiler({ config, keepSourceMaps: false }, null); + config.coverageAnalysis = 'off'; + const input = [textFile({ mutated: true }), binaryFile({ mutated: true }), webFile({ mutated: true })]; + const output = sut.transpile(input); + expect(output.error).null; + expect(output.outputFiles).deep.eq(input); + }); + + describe('when coverage analysis is "all"', () => { + + beforeEach(() => { + config.coverageAnalysis = 'all'; + sut = new CoverageInstrumenterTranspiler({ config, keepSourceMaps: false }, null); + }); + + it('should instrument code of mutated files', () => { + const input = [ + textFile({ mutated: true, content: 'function something() {}' }), + binaryFile({ mutated: true }), + webFile({ mutated: true }), + textFile({ mutated: false }) + ]; + const output = sut.transpile(input); + expect(output.error).null; + expect(output.outputFiles[1]).eq(output.outputFiles[1]); + expect(output.outputFiles[2]).eq(output.outputFiles[2]); + expect(output.outputFiles[3]).eq(output.outputFiles[3]); + const instrumentedContent = (output.outputFiles[0] as TextFile).content; + expect(instrumentedContent).to.contain('function something(){__cov_').and.contain('.f[\'1\']++'); + }); + + it('should create a statement map for mutated files', () => { + const input = [ + textFile({ name: 'something.js', mutated: true, content: 'function something () {}' }), + textFile({ name: 'foobar.js', mutated: true, content: 'console.log("foobar");' }) + ]; + sut.transpile(input); + expect(sut.statementMapsPerFile).deep.eq({ + 'something.js': { '1': { start: { line: 0, column: 0 }, end: { line: 0, column: 24 } } }, + 'foobar.js': { '1': { start: { line: 0, column: 0 }, end: { line: 0, column: 22 } } } + }); + }); + + it('should fill error message and not transpile input when the file contains a parse error', () => { + const invalidJavascriptFile = textFile({ name: 'invalid/file.js', content: 'function something {}', mutated: true }); + const output = sut.transpile([invalidJavascriptFile]); + expect(output.error).contains('Could not instrument "invalid/file.js" for code coverage. Error: Line 1: Unexpected token {'); + }); + }); + + describe('when coverage analysis is "perTest" and there is a testFramework', () => { + let input: TextFile[]; + + beforeEach(() => { + config.coverageAnalysis = 'perTest'; + sut = new CoverageInstrumenterTranspiler({ config, keepSourceMaps: false }, testFramework()); + input = [textFile({ mutated: true, content: 'function something() {}' })]; + }); + + it('should use the coverage variable "__strykerCoverageCurrentTest__"', () => { + const output = sut.transpile(input); + expect(output.error).null; + const instrumentedContent = (output.outputFiles[1] as TextFile).content; + expect(instrumentedContent).to.contain('__strykerCoverageCurrentTest__').and.contain('.f[\'1\']++'); + }); + + it('should also add a collectCoveragePerTest file', () => { + const output = sut.transpile(input); + expect(output.error).null; + expect(output.outputFiles).lengthOf(2); + const actualContent = (output.outputFiles[0] as TextFile).content; + expect(actualContent).to.have.length.greaterThan(30); + expect(actualContent).to.contain('beforeEach()'); + expect(actualContent).to.contain('afterEach()'); + }); + }); + + it('should result in an error if coverage analysis is "perTest" and there is no testFramework', () => { + config.coverageAnalysis = 'perTest'; + sut = new CoverageInstrumenterTranspiler({ config, keepSourceMaps: true }, null); + const output = sut.transpile([textFile({ content: 'a + b' })]); + expect(output.error).eq('Cannot measure coverage results per test, there is no testFramework and thus no way of executing code right before and after each test.'); + }); +}); \ No newline at end of file diff --git a/packages/stryker/test/unit/transpiler/TranspilerFacadeSpec.ts b/packages/stryker/test/unit/transpiler/TranspilerFacadeSpec.ts index df0f67f09b..0df349f66e 100644 --- a/packages/stryker/test/unit/transpiler/TranspilerFacadeSpec.ts +++ b/packages/stryker/test/unit/transpiler/TranspilerFacadeSpec.ts @@ -42,9 +42,10 @@ describe('TranspilerFacade', () => { let resultTwo: TranspileResult; let locationOne: FileLocation; let locationTwo: FileLocation; + let config: Config; beforeEach(() => { - const config = new Config(); + config = new Config(); config.transpilers.push('transpiler-one', 'transpiler-two'); transpilerOne = mock(TranspilerFacade); transpilerTwo = mock(TranspilerFacade); @@ -59,16 +60,17 @@ describe('TranspilerFacade', () => { transpilerOne.getMappedLocation.returns(locationOne); transpilerTwo.transpile.returns(resultTwo); transpilerTwo.getMappedLocation.returns(locationTwo); - sut = new TranspilerFacade({ config, keepSourceMaps: true }); }); it('should create two transpilers', () => { + sut = new TranspilerFacade({ config, keepSourceMaps: true }); expect(createStub).calledTwice; expect(createStub).calledWith('transpiler-one'); expect(createStub).calledWith('transpiler-two'); }); it('should chain the transpilers when `transpile` is called', () => { + sut = new TranspilerFacade({ config, keepSourceMaps: true }); const input = [file({ name: 'input' })]; const result = sut.transpile(input); expect(result).eq(resultTwo); @@ -76,7 +78,23 @@ describe('TranspilerFacade', () => { expect(transpilerTwo.transpile).calledWith(resultOne.outputFiles); }); + it('should chain an additional transpiler when requested', () => { + const additionalTranspiler = mock(TranspilerFacade); + const expectedResult = transpileResult({ outputFiles: [file({ name: 'result-3' })] }); + additionalTranspiler.transpile.returns(expectedResult); + const input = [file({ name: 'input' })]; + sut = new TranspilerFacade( + { config, keepSourceMaps: true }, + { name: 'someTranspiler', transpiler: additionalTranspiler } + ); + const output = sut.transpile(input); + expect(output).eq(expectedResult); + expect(additionalTranspiler.transpile).calledWith(resultTwo.outputFiles); + }); + + it('should stop chaining if an error occurs during `transpile`', () => { + sut = new TranspilerFacade({ config, keepSourceMaps: true }); const input = [file({ name: 'input' })]; resultOne.error = 'an error'; const result = sut.transpile(input); @@ -86,6 +104,7 @@ describe('TranspilerFacade', () => { }); it('should chain the transpilers when `getMappedLocation` is called', () => { + sut = new TranspilerFacade({ config, keepSourceMaps: true }); const input = fileLocation({ fileName: 'input' }); const result = sut.getMappedLocation(input); expect(result).eq(locationTwo);