diff --git a/packages/api/src/test_framework/TestFramework.ts b/packages/api/src/test_framework/TestFramework.ts index e64b4e4560..4da00e02a1 100644 --- a/packages/api/src/test_framework/TestFramework.ts +++ b/packages/api/src/test_framework/TestFramework.ts @@ -18,8 +18,8 @@ interface TestFramework { /** * Creates a code fragment which, if included in a test run, - * will be responsible for filtering out tests with given ids. - * The first test gets id 0, the second id 1, etc. + * will be responsible for filtering out tests with given test selector. + * If te test selection array is empty it should reset the filtering for the next test run. * * @param selections A list indicating the tests to select. * @returns A script which, if included in the test run, will filter out the correct tests. diff --git a/packages/core/src/Sandbox.ts b/packages/core/src/Sandbox.ts index f3c7b320be..9587343f31 100644 --- a/packages/core/src/Sandbox.ts +++ b/packages/core/src/Sandbox.ts @@ -197,8 +197,12 @@ export default class Sandbox { } private getFilterTestsHooks(mutant: TestableMutant): string | undefined { - if (this.testFramework && !mutant.runAllTests) { - return wrapInClosure(this.testFramework.filter(mutant.selectedTests)); + if (this.testFramework) { + if (mutant.runAllTests) { + return wrapInClosure(this.testFramework.filter([])); + } else { + return wrapInClosure(this.testFramework.filter(mutant.selectedTests)); + } } else { return undefined; } diff --git a/packages/core/src/child-proxy/ChildProcessProxy.ts b/packages/core/src/child-proxy/ChildProcessProxy.ts index bc80619aa3..876201d399 100644 --- a/packages/core/src/child-proxy/ChildProcessProxy.ts +++ b/packages/core/src/child-proxy/ChildProcessProxy.ts @@ -167,6 +167,10 @@ export default class ChildProcessProxy implements Disposable { } } + public get stdout() { + return this.stdoutBuilder.toString(); + } + private reportError(error: Error) { this.workerTasks.filter(task => !task.isCompleted).forEach(task => task.reject(error)); } diff --git a/packages/core/test/integration/child-proxy/ChildProcessProxy.it.spec.ts b/packages/core/test/integration/child-proxy/ChildProcessProxy.it.spec.ts index b1d0add8db..37ff5ce319 100644 --- a/packages/core/test/integration/child-proxy/ChildProcessProxy.it.spec.ts +++ b/packages/core/test/integration/child-proxy/ChildProcessProxy.it.spec.ts @@ -3,18 +3,15 @@ import * as path from 'path'; import { File, LogLevel } from '@stryker-mutator/api/core'; import { Logger } from '@stryker-mutator/api/logging'; import { commonTokens } from '@stryker-mutator/api/plugin'; -import { testInjector } from '@stryker-mutator/test-helpers'; +import { testInjector, LoggingServer } from '@stryker-mutator/test-helpers'; import { expect } from 'chai'; import * as log4js from 'log4js'; import { filter } from 'rxjs/operators'; -import getPort = require('get-port'); - import ChildProcessCrashedError from '../../../src/child-proxy/ChildProcessCrashedError'; import ChildProcessProxy from '../../../src/child-proxy/ChildProcessProxy'; import OutOfMemoryError from '../../../src/child-proxy/OutOfMemoryError'; import { Task } from '../../../src/utils/Task'; -import LoggingServer from '../../helpers/LoggingServer'; import currentLogMock from '../../helpers/logMock'; import { Mock } from '../../helpers/producers'; import { sleep } from '../../helpers/testUtils'; @@ -29,10 +26,10 @@ describe(ChildProcessProxy.name, () => { const workingDir = '..'; beforeEach(async () => { - const port = await getPort(); + loggingServer = new LoggingServer(); + const port = await loggingServer.listen(); const options = testInjector.injector.resolve(commonTokens.options); log = currentLogMock(); - loggingServer = new LoggingServer(port); sut = ChildProcessProxy.create(require.resolve('./Echo'), { port, level: LogLevel.Debug }, options, { name: echoName }, workingDir, Echo); }); diff --git a/packages/core/test/integration/test-runner/ResilientTestRunnerFactory.it.spec.ts b/packages/core/test/integration/test-runner/ResilientTestRunnerFactory.it.spec.ts index 1ceb6ffe7d..d16816febb 100644 --- a/packages/core/test/integration/test-runner/ResilientTestRunnerFactory.it.spec.ts +++ b/packages/core/test/integration/test-runner/ResilientTestRunnerFactory.it.spec.ts @@ -6,12 +6,10 @@ import { strykerOptions } from '@stryker-mutator/test-helpers/src/factory'; import { expect } from 'chai'; import * as log4js from 'log4js'; import { toArray } from 'rxjs/operators'; - -import getPort = require('get-port'); +import { LoggingServer } from '@stryker-mutator/test-helpers'; import LoggingClientContext from '../../../src/logging/LoggingClientContext'; import ResilientTestRunnerFactory from '../../../src/test-runner/ResilientTestRunnerFactory'; -import LoggingServer from '../../helpers/LoggingServer'; import { sleep } from '../../helpers/testUtils'; describe('ResilientTestRunnerFactory integration', () => { @@ -25,8 +23,8 @@ describe('ResilientTestRunnerFactory integration', () => { beforeEach(async () => { // Make sure there is a logging server listening - const port = await getPort(); - loggingServer = new LoggingServer(port); + loggingServer = new LoggingServer(); + const port = await loggingServer.listen(); loggingContext = { port, level: LogLevel.Trace }; options = strykerOptions({ plugins: [require.resolve('./AdditionalTestRunners')], diff --git a/packages/core/test/unit/Sandbox.spec.ts b/packages/core/test/unit/Sandbox.spec.ts index ce565988bf..0face9dd00 100644 --- a/packages/core/test/unit/Sandbox.spec.ts +++ b/packages/core/test/unit/Sandbox.spec.ts @@ -37,12 +37,11 @@ describe(Sandbox.name, () => { let options: Config; let inputFiles: File[]; let testRunner: Mock; - let testFrameworkStub: any; + let testFrameworkStub: sinon.SinonStubbedInstance; let expectedFileToMutate: File; let notMutatedFile: File; let sandboxDirectory: string; let expectedTargetFileToMutate: string; - let expectedTestFrameworkHooksFile: string; let writeFileStub: sinon.SinonStub; let symlinkJunctionStub: sinon.SinonStub; let findNodeModulesStub: sinon.SinonStub; @@ -55,13 +54,14 @@ describe(Sandbox.name, () => { options = { timeoutFactor: 23, timeoutMS: 1000, testRunner: 'sandboxUnitTestRunner', symlinkNodeModules: true } as any; testRunner = { init: sinon.stub(), run: sinon.stub().resolves(runResult), dispose: sinon.stub() }; testFrameworkStub = { - filter: sinon.stub() + filter: sinon.stub(), + afterEach: sinon.stub(), + beforeEach: sinon.stub() }; expectedFileToMutate = new File(path.resolve('file1'), 'original code'); notMutatedFile = new File(path.resolve('file2'), 'to be mutated'); sandboxDirectory = path.resolve('random-folder-3'); expectedTargetFileToMutate = path.join(sandboxDirectory, 'file1'); - expectedTestFrameworkHooksFile = path.join(sandboxDirectory, '___testHooksForStryker.js'); inputFiles = [expectedFileToMutate, notMutatedFile]; temporaryDirectoryMock = sinon.createStubInstance(TemporaryDirectory); temporaryDirectoryMock.createRandomDirectory.returns(sandboxDirectory); @@ -84,12 +84,12 @@ describe(Sandbox.name, () => { files: readonly File[]; } function createSut(overrides?: Partial) { - const args: CreateArgs = { + const { files, testFramework, overheadTimeMS }: CreateArgs = { files: inputFiles, overheadTimeMS: OVERHEAD_TIME_MS, - testFramework: null + testFramework: null, + ...overrides }; - const { files, testFramework, overheadTimeMS } = { ...args, ...overrides }; return Sandbox.create(options, SANDBOX_INDEX, files, testFramework, overheadTimeMS, LOGGING_CONTEXT, temporaryDirectoryMock as any); } @@ -112,19 +112,19 @@ describe(Sandbox.name, () => { }); it('should have created a sandbox folder', async () => { - await createSut(testFrameworkStub); + await createSut({ testFramework: testFrameworkStub }); expect(temporaryDirectoryMock.createRandomDirectory).calledWith('sandbox'); }); it('should symlink node modules in sandbox directory if exists', async () => { - await createSut(testFrameworkStub); + await createSut({ testFramework: testFrameworkStub }); expect(findNodeModulesStub).calledWith(process.cwd()); expect(symlinkJunctionStub).calledWith('node_modules', path.join(sandboxDirectory, 'node_modules')); }); it('should not symlink node modules in sandbox directory if no node_modules exist', async () => { findNodeModulesStub.resolves(null); - await createSut(testFrameworkStub); + await createSut({ testFramework: testFrameworkStub }); expect(log.warn).calledWithMatch('Could not find a node_modules'); expect(log.warn).calledWithMatch(process.cwd()); expect(symlinkJunctionStub).not.called; @@ -133,7 +133,7 @@ describe(Sandbox.name, () => { it('should log a warning if "node_modules" already exists in the working folder', async () => { findNodeModulesStub.resolves('node_modules'); symlinkJunctionStub.rejects(fileAlreadyExistsError()); - await createSut(testFrameworkStub); + await createSut({ testFramework: testFrameworkStub }); expect(log.warn).calledWithMatch( normalizeWhitespaces( `Could not symlink "node_modules" in sandbox directory, it is already created in the sandbox. @@ -147,7 +147,7 @@ describe(Sandbox.name, () => { findNodeModulesStub.resolves('basePath/node_modules'); const error = new Error('unknown'); symlinkJunctionStub.rejects(error); - await createSut(testFrameworkStub); + await createSut({ testFramework: testFrameworkStub }); expect(log.warn).calledWithMatch( normalizeWhitespaces('Unexpected error while trying to symlink "basePath/node_modules" in sandbox directory.'), error @@ -156,7 +156,7 @@ describe(Sandbox.name, () => { it('should symlink node modules in sandbox directory if `symlinkNodeModules` is `false`', async () => { options.symlinkNodeModules = false; - await createSut(testFrameworkStub); + await createSut({ testFramework: testFrameworkStub }); expect(symlinkJunctionStub).not.called; expect(findNodeModulesStub).not.called; }); @@ -229,6 +229,7 @@ describe(Sandbox.name, () => { const sut = await createSut({ testFramework: testFrameworkStub }); await sut.runMutant(transpiledMutant); expect(testFrameworkStub.filter).to.have.been.calledWith(transpiledMutant.mutant.selectedTests); + expect(testRunner.run).calledWithMatch({ testHooks: wrapInClosure(testFilterCodeFragment) }); }); it('should provide the filter code as testHooks, correct timeout and mutatedFileName', async () => { @@ -254,32 +255,25 @@ describe(Sandbox.name, () => { }); it('should not filter any tests when testFramework = null', async () => { + transpiledMutant.mutant.selectAllTests(runResult, TestSelectionResult.Success); const sut = await createSut(); - const mutant = new TestableMutant('2', createMutant(), new SourceFile(new File('', ''))); - await sut.runMutant(new TranspiledMutant(mutant, { outputFiles: [new File(expectedTargetFileToMutate, '')], error: null }, true)); - expect(fileUtils.writeFile).not.calledWith(expectedTestFrameworkHooksFile); + await sut.runMutant(transpiledMutant); + expect(testRunner.run).calledWithMatch({ testHooks: undefined }); }); - it('should not filter any tests when runAllTests = true', async () => { + it('should filter no tests when runAllTests = true', async () => { // Arrange while (transpiledMutant.mutant.selectedTests.pop()); transpiledMutant.mutant.selectAllTests(runResult, TestSelectionResult.Success); - const sut = await createSut(); + testFrameworkStub.filter.returns('empty filter'); + const sut = await createSut({ testFramework: testFrameworkStub }); // Act await sut.runMutant(transpiledMutant); // Assert - expect(fileUtils.writeFile).not.calledWith(expectedTestFrameworkHooksFile); - expect(testRunner.run).called; - }); - - it('should not filter any tests when runAllTests = true', async () => { - const sut = await createSut(); - const mutant = new TestableMutant('2', createMutant(), new SourceFile(new File('', ''))); - mutant.selectAllTests(runResult, TestSelectionResult.Failed); - sut.runMutant(new TranspiledMutant(mutant, { outputFiles: [new File(expectedTargetFileToMutate, '')], error: null }, true)); - expect(fileUtils.writeFile).not.calledWith(expectedTestFrameworkHooksFile); + expect(testRunner.run).calledWithMatch({ testHooks: wrapInClosure('empty filter') }); + expect(testFrameworkStub.filter).calledWith([]); }); it('should report a runtime error when test run errored', async () => { diff --git a/packages/jasmine-framework/src/JasmineTestFramework.ts b/packages/jasmine-framework/src/JasmineTestFramework.ts index 55ea7049a8..6f0d1b5baf 100644 --- a/packages/jasmine-framework/src/JasmineTestFramework.ts +++ b/packages/jasmine-framework/src/JasmineTestFramework.ts @@ -28,10 +28,14 @@ export default class JasmineTestFramework implements TestFramework { } public filter(testSelections: TestSelection[]): string { - const names = testSelections.map(selection => selection.name); - return ` + if (testSelections.length) { + const names = testSelections.map(selection => selection.name); + return ` jasmine.getEnv().specFilter = function (spec) { return ${JSON.stringify(names)}.indexOf(spec.getFullName()) !== -1; }`; + } else { + return ''; + } } } diff --git a/packages/jasmine-framework/test/unit/JasmineTestFramework.spec.ts b/packages/jasmine-framework/test/unit/JasmineTestFramework.spec.ts index 6a4c794e67..1c11efd486 100644 --- a/packages/jasmine-framework/test/unit/JasmineTestFramework.spec.ts +++ b/packages/jasmine-framework/test/unit/JasmineTestFramework.spec.ts @@ -21,7 +21,7 @@ describe('JasmineTestFramework', () => { }); describe('filter()', () => { - it("should result in a specFilter of jasmine it's", () => + it("should result in a specFilter of jasmine it's", () => { expect( sut.filter([ { id: 5, name: 'test five' }, @@ -29,6 +29,11 @@ describe('JasmineTestFramework', () => { ]) ) .to.contain('jasmine.getEnv().specFilter = function (spec)') - .and.to.contain('return ["test five","test eight"].indexOf(spec.getFullName()) !== -1;')); + .and.to.contain('return ["test five","test eight"].indexOf(spec.getFullName()) !== -1;'); + }); + + it('should result in an empty string when filter is empty', () => { + expect(sut.filter([])).eq(''); + }); }); }); diff --git a/packages/jasmine-runner/test/integration/JasmineRunner.it.spec.ts b/packages/jasmine-runner/test/integration/JasmineRunner.it.spec.ts index 884ce2fb3b..3c35f7913b 100644 --- a/packages/jasmine-runner/test/integration/JasmineRunner.it.spec.ts +++ b/packages/jasmine-runner/test/integration/JasmineRunner.it.spec.ts @@ -99,6 +99,17 @@ describe('JasmineRunner integration', () => { expectTestsFiltered(runResult.tests, 1, 3); }); + it('should be able to clear the filter after a filtered run', async () => { + // Arrange + const testFramework = new JasmineTestFramework(); + const filter1Test = wrapInClosure(testFramework.filter([{ id: 1, name: expectedJasmineInitResults[1].name }])); + const filterNoTests = wrapInClosure(testFramework.filter([])); + + await sut.run({ testHooks: filter1Test }); + const actualResult = await sut.run({ testHooks: filterNoTests }); + expectTestResultsToEqual(actualResult.tests, expectedJasmineInitResults); + }); + it('should be able to filter tests in quick succession', async () => { // Arrange const testFramework = new JasmineTestFramework(); diff --git a/packages/karma-runner/package.json b/packages/karma-runner/package.json index 8c0e55f2c6..6360256b31 100644 --- a/packages/karma-runner/package.json +++ b/packages/karma-runner/package.json @@ -5,7 +5,9 @@ "main": "src/index.js", "scripts": { "test": "nyc --exclude-after-remap=false --check-coverage --reporter=html --report-dir=reports/coverage --lines 80 --functions 80 --branches 75 npm run mocha", - "mocha": "mocha \"test/helpers/**/*.js\" \"test/unit/**/*.js\" && mocha --timeout 30000 --exit \"test/helpers/**/*.js\" \"test/integration/**/*.js\"", + "mocha": "npm run test:unit && npm run test:integration", + "test:unit": "mocha \"test/helpers/**/*.js\" \"test/unit/**/*.js\"", + "test:integration": "mocha --timeout 30000 --exit \"test/helpers/**/*.js\" \"test/integration/**/*.js\"", "stryker": "node ../core/bin/stryker run" }, "repository": { @@ -37,8 +39,10 @@ "@types/semver": "~7.1.0", "jasmine-core": "~3.5.0", "karma": "~4.1.0", + "karma-chai": "^0.1.0", "karma-jasmine": "~3.0.1", - "karma-phantomjs-launcher": "~1.0.4" + "karma-mocha": "^1.3.0", + "karma-phantomjs-launcher": "^1.0.4" }, "peerDependencies": { "@stryker-mutator/core": "^2.0.0" diff --git a/packages/karma-runner/test/integration/KarmaTestRunner.it.spec.ts b/packages/karma-runner/test/integration/KarmaTestRunner.it.spec.ts index c1567f0315..bcc09f7b92 100644 --- a/packages/karma-runner/test/integration/KarmaTestRunner.it.spec.ts +++ b/packages/karma-runner/test/integration/KarmaTestRunner.it.spec.ts @@ -6,9 +6,11 @@ import JasmineTestFramework from '@stryker-mutator/jasmine-framework/src/Jasmine import { testInjector } from '@stryker-mutator/test-helpers'; import { expect } from 'chai'; import { FilePattern } from 'karma'; +import { TestSelection } from '@stryker-mutator/api/test_framework'; import KarmaTestRunner from '../../src/KarmaTestRunner'; import { expectTestResults } from '../helpers/assertions'; +import MochaTestFramework from '../../../mocha-framework/src/MochaTestFramework'; function wrapInClosure(codeFragment: string) { return ` @@ -17,7 +19,13 @@ function wrapInClosure(codeFragment: string) { })((Function('return this'))());`; } -function setOptions(files: ReadonlyArray, coverageAnalysis: 'all' | 'perTest' | 'off' = 'off'): void { +function setOptions( + files: ReadonlyArray = [ + 'testResources/sampleProject/src-instrumented/Add.js', + 'testResources/sampleProject/test-jasmine/AddSpec.js' + ], + coverageAnalysis: 'all' | 'perTest' | 'off' = 'off' +): void { testInjector.options.coverageAnalysis = coverageAnalysis; testInjector.options.karma = { config: { @@ -32,7 +40,7 @@ function createSut() { return testInjector.injector.injectClass(KarmaTestRunner); } -describe('KarmaTestRunner', () => { +describe(`${KarmaTestRunner.name} integration`, () => { let sut: KarmaTestRunner; const expectToHaveSuccessfulTests = (result: RunResult, n: number) => { @@ -47,10 +55,54 @@ describe('KarmaTestRunner', () => { }); }; + describe('with mocha', () => { + let testFramework: MochaTestFramework; + + const test0: Readonly = Object.freeze({ + id: 0, + name: 'Add should be able to add two numbers' + }); + + const test3: Readonly = { + id: 3, + name: 'Add should be able to recognize a negative number' + }; + + before(() => { + testFramework = new MochaTestFramework(); + setOptions(['testResources/sampleProject/src/Add.js', 'testResources/sampleProject/test-mocha/AddSpec.js']); + testInjector.options.karma.config.frameworks = ['mocha', 'chai']; + sut = createSut(); + return sut.init(); + }); + + it('should report completed tests', async () => { + const runResult = await sut.run({}); + // await new Promise(res => {}); + expectToHaveSuccessfulTests(runResult, 5); + expectToHaveFailedTests(runResult, []); + expect(runResult.status).to.be.eq(RunStatus.Complete); + }); + + it('should be able to filter tests', async () => { + const testHooks = wrapInClosure(testFramework.filter([test0, test3])); + const actualResult = await sut.run({ testHooks }); + expectToHaveSuccessfulTests(actualResult, 2); + expect(actualResult.tests[0].name).eq(test0.name); + expect(actualResult.tests[1].name).eq(test3.name); + }); + + it('should be able to clear the filter after a filtered run', async () => { + await sut.run({ testHooks: wrapInClosure(testFramework.filter([test0, test3])) }); + const actualResult = await sut.run({ testHooks: wrapInClosure(testFramework.filter([])) }); + expect(actualResult.tests).lengthOf(5); + }); + }); + describe('when all tests succeed', () => { describe('with simple add function to test', () => { before(() => { - setOptions(['testResources/sampleProject/src/Add.js', 'testResources/sampleProject/test/AddSpec.js']); + setOptions(['testResources/sampleProject/src/Add.js', 'testResources/sampleProject/test-jasmine/AddSpec.js']); sut = createSut(); return sut.init(); }); @@ -90,8 +142,8 @@ describe('KarmaTestRunner', () => { before(() => { setOptions([ 'testResources/sampleProject/src/Add.js', - 'testResources/sampleProject/test/AddSpec.js', - 'testResources/sampleProject/test/AddFailedSpec.js' + 'testResources/sampleProject/test-jasmine/AddSpec.js', + 'testResources/sampleProject/test-jasmine/AddFailedSpec.js' ]); sut = createSut(); return sut.init(); @@ -112,7 +164,7 @@ describe('KarmaTestRunner', () => { setOptions([ 'testResources/sampleProject/src/Add.js', 'testResources/sampleProject/src/Error.js', - 'testResources/sampleProject/test/AddSpec.js' + 'testResources/sampleProject/test-jasmine/AddSpec.js' ]); sut = createSut(); return sut.init(); @@ -128,7 +180,7 @@ describe('KarmaTestRunner', () => { describe('when no error occurred and no test is performed', () => { before(() => { - setOptions(['testResources/sampleProject/src/Add.js', 'testResources/sampleProject/test/EmptySpec.js']); + setOptions(['testResources/sampleProject/src/Add.js', 'testResources/sampleProject/test-jasmine/EmptySpec.js']); sut = createSut(); return sut.init(); }); @@ -149,7 +201,7 @@ describe('KarmaTestRunner', () => { before(() => { setOptions([ { pattern: 'testResources/sampleProject/src/Add.js', included: true }, - { pattern: 'testResources/sampleProject/test/AddSpec.js', included: true }, + { pattern: 'testResources/sampleProject/test-jasmine/AddSpec.js', included: true }, { pattern: 'testResources/sampleProject/src/Error.js', included: false } ]); sut = createSut(); @@ -166,7 +218,7 @@ describe('KarmaTestRunner', () => { describe('when coverage data is available', () => { before(() => { - setOptions(['testResources/sampleProject/src-instrumented/Add.js', 'testResources/sampleProject/test/AddSpec.js'], 'all'); + setOptions(['testResources/sampleProject/src-instrumented/Add.js', 'testResources/sampleProject/test-jasmine/AddSpec.js'], 'all'); sut = createSut(); return sut.init(); }); @@ -187,7 +239,8 @@ describe('KarmaTestRunner', () => { before(async () => { dummyServer = await DummyServer.create(); - sut = testInjector.injector.injectClass(KarmaTestRunner); + setOptions(); + sut = createSut(); return sut.init(); }); diff --git a/packages/karma-runner/testResources/sampleProject/karma.conf.js b/packages/karma-runner/testResources/sampleProject/karma.conf.js index 069da0d5ba..44a0bed8f8 100644 --- a/packages/karma-runner/testResources/sampleProject/karma.conf.js +++ b/packages/karma-runner/testResources/sampleProject/karma.conf.js @@ -4,7 +4,7 @@ module.exports = function (config) { config.set({ files: [ __dirname + '/src/*.js', - __dirname + '/test/*.js' + __dirname + '/test-jasmine/*.js' ], exclude: [ __dirname + '/src/Error.js', @@ -19,4 +19,4 @@ module.exports = function (config) { 'PhantomJS' ] }); -} \ No newline at end of file +} diff --git a/packages/karma-runner/testResources/sampleProject/test/AddFailedSpec.js b/packages/karma-runner/testResources/sampleProject/test-jasmine/AddFailedSpec.js similarity index 100% rename from packages/karma-runner/testResources/sampleProject/test/AddFailedSpec.js rename to packages/karma-runner/testResources/sampleProject/test-jasmine/AddFailedSpec.js diff --git a/packages/karma-runner/testResources/sampleProject/test/AddSpec.js b/packages/karma-runner/testResources/sampleProject/test-jasmine/AddSpec.js similarity index 100% rename from packages/karma-runner/testResources/sampleProject/test/AddSpec.js rename to packages/karma-runner/testResources/sampleProject/test-jasmine/AddSpec.js diff --git a/packages/karma-runner/testResources/sampleProject/test/CircleSpec.js b/packages/karma-runner/testResources/sampleProject/test-jasmine/CircleSpec.js similarity index 100% rename from packages/karma-runner/testResources/sampleProject/test/CircleSpec.js rename to packages/karma-runner/testResources/sampleProject/test-jasmine/CircleSpec.js diff --git a/packages/karma-runner/testResources/sampleProject/test/EmptySpec.js b/packages/karma-runner/testResources/sampleProject/test-jasmine/EmptySpec.js similarity index 100% rename from packages/karma-runner/testResources/sampleProject/test/EmptySpec.js rename to packages/karma-runner/testResources/sampleProject/test-jasmine/EmptySpec.js diff --git a/packages/karma-runner/testResources/sampleProject/test/FailingAddSpec.js b/packages/karma-runner/testResources/sampleProject/test-jasmine/FailingAddSpec.js similarity index 100% rename from packages/karma-runner/testResources/sampleProject/test/FailingAddSpec.js rename to packages/karma-runner/testResources/sampleProject/test-jasmine/FailingAddSpec.js diff --git a/packages/karma-runner/testResources/sampleProject/test-mocha/AddSpec.js b/packages/karma-runner/testResources/sampleProject/test-mocha/AddSpec.js new file mode 100644 index 0000000000..7a2f8ad628 --- /dev/null +++ b/packages/karma-runner/testResources/sampleProject/test-mocha/AddSpec.js @@ -0,0 +1,48 @@ +describe('Add', function () { + it('should be able to add two numbers', function () { + var num1 = 2; + var num2 = 5; + var expected = num1 + num2; + + var actual = add(num1, num2); + + expect(actual).eq(expected); + window.setTimeout(function () { + done(); + }, 20); + }); + + it('should be able 1 to a number', function () { + var number = 2; + var expected = 3; + + var actual = addOne(number); + + expect(actual).eq(expected); + }); + + it('should be able negate a number', function () { + var number = 2; + var expected = -2; + + var actual = negate(number); + + expect(actual).eq(expected); + }); + + it('should be able to recognize a negative number', function () { + var number = -2; + + var isNegative = isNegativeNumber(number); + + expect(isNegative).eq(true); + }); + + it('should be able to recognize that 0 is not a negative number', function () { + var number = 0; + + var isNegative = isNegativeNumber(number); + + expect(isNegative).eq(false); + }); +}); diff --git a/packages/karma-runner/tsconfig.test.json b/packages/karma-runner/tsconfig.test.json index 1a212580d7..ae773b007d 100644 --- a/packages/karma-runner/tsconfig.test.json +++ b/packages/karma-runner/tsconfig.test.json @@ -18,6 +18,9 @@ }, { "path": "../test-helpers/tsconfig.src.json" + }, + { + "path": "../mocha-framework/tsconfig.src.json" } ] -} \ No newline at end of file +} diff --git a/packages/mocha-framework/package.json b/packages/mocha-framework/package.json index 95c721b731..4ea707bd69 100644 --- a/packages/mocha-framework/package.json +++ b/packages/mocha-framework/package.json @@ -43,6 +43,7 @@ "mocha": ">= 2.3.3 < 7" }, "dependencies": { - "@stryker-mutator/api": "^2.5.0" + "@stryker-mutator/api": "^2.5.0", + "@stryker-mutator/util": "^2.5.0" } } diff --git a/packages/mocha-framework/src/MochaTestFramework.ts b/packages/mocha-framework/src/MochaTestFramework.ts index e83840d9e8..721c736658 100644 --- a/packages/mocha-framework/src/MochaTestFramework.ts +++ b/packages/mocha-framework/src/MochaTestFramework.ts @@ -1,5 +1,14 @@ import { TestFramework, TestSelection } from '@stryker-mutator/api/test_framework'; +const FILTER_HEADER_FRAGMENT = ` + var Mocha = window.Mocha || require('mocha'); + var describe = Mocha.describe;`; +const FILTER_RESET = `${FILTER_HEADER_FRAGMENT} + if (window.____mochaAddTest) { + Mocha.Suite.prototype.addTest = window.____mochaAddTest; + } +`; + export default class MochaTestFramework implements TestFramework { public beforeEach(codeFragment: string): string { return `beforeEach(function() { @@ -14,9 +23,9 @@ export default class MochaTestFramework implements TestFramework { } public filter(testSelections: TestSelection[]) { - const selectedTestNames = testSelections.map(selection => selection.name); - return `var Mocha = window.Mocha || require('mocha'); - var describe = Mocha.describe; + if (testSelections.length) { + const selectedTestNames = testSelections.map(selection => selection.name); + return `${FILTER_HEADER_FRAGMENT} var selectedTestNames = ${JSON.stringify(selectedTestNames)}; if (window.____mochaAddTest) { Mocha.Suite.prototype.addTest = window.____mochaAddTest; @@ -28,9 +37,12 @@ export default class MochaTestFramework implements TestFramework { Mocha.Suite.prototype.addTest = function (test) { // Only actually add the tests with the expected names var name = this.fullTitle() + ' ' + test.title; - if(selectedTestNames.indexOf(name) !== -1) { + if(!selectedTestNames.length || selectedTestNames.indexOf(name) !== -1) { realAddTest.apply(this, arguments); } };`; + } else { + return FILTER_RESET; + } } } diff --git a/packages/mocha-framework/test/unit/MochaTestFramework.spec.ts b/packages/mocha-framework/test/unit/MochaTestFramework.spec.ts index 26d60e9335..32d42a4709 100644 --- a/packages/mocha-framework/test/unit/MochaTestFramework.spec.ts +++ b/packages/mocha-framework/test/unit/MochaTestFramework.spec.ts @@ -1,4 +1,5 @@ import { expect } from 'chai'; +import { normalizeWhitespaces } from '@stryker-mutator/util'; import MochaTestFramework from '../../src/MochaTestFramework'; @@ -29,9 +30,21 @@ describe('MochaTestFramework', () => { ]) ) .to.contain('var realAddTest = Mocha.Suite.prototype.addTest;') + .to.contain("var Mocha = window.Mocha || require('mocha');") .and.to.contain('selectedTestNames = ["test five","test eight"];') - .and.to.contain('if(selectedTestNames.indexOf(name) !== -1)') + .and.to.contain('if(!selectedTestNames.length || selectedTestNames.indexOf(name) !== -1)') .and.to.contain('realAddTest.apply(this, arguments);'); }); + it('should result in a filter reset when selecting no tests', () => { + const actualFilter = normalizeWhitespaces(sut.filter([])); + const expectedFilter = normalizeWhitespaces(` + var Mocha = window.Mocha || require('mocha'); + var describe = Mocha.describe; + if (window.____mochaAddTest) { + Mocha.Suite.prototype.addTest = window.____mochaAddTest; + } + `); + expect(actualFilter).eq(expectedFilter); + }); }); }); diff --git a/packages/mocha-framework/tsconfig.test.json b/packages/mocha-framework/tsconfig.test.json index 6931176018..d7eab70bef 100644 --- a/packages/mocha-framework/tsconfig.test.json +++ b/packages/mocha-framework/tsconfig.test.json @@ -13,6 +13,9 @@ "references": [ { "path": "./tsconfig.src.json" + }, + { + "path": "../util/tsconfig.src.json" } ] -} \ No newline at end of file +} diff --git a/packages/mocha-runner/package.json b/packages/mocha-runner/package.json index e413549fa6..4c8b543a69 100644 --- a/packages/mocha-runner/package.json +++ b/packages/mocha-runner/package.json @@ -5,7 +5,9 @@ "main": "src/index.js", "scripts": { "test": "nyc --exclude-after-remap=false --check-coverage --reporter=html --report-dir=reports/coverage --lines 80 --functions 80 --branches 75 npm run mocha", - "mocha": "mocha \"test/helpers/**/*.js\" \"test/unit/**/*.js\" && mocha --timeout 10000 \"test/helpers/**/*.js\" \"test/integration/**/*.js\"", + "mocha": "npm run test:unit && npm run test:integration", + "test:unit": "mocha \"test/helpers/**/*.js\" \"test/unit/**/*.js\"", + "test:integration": "mocha --no-timeout \"test/helpers/**/*.js\" \"test/integration/**/*.js\"", "stryker": "node ../core/bin/stryker run" }, "repository": { @@ -41,7 +43,7 @@ "devDependencies": { "@stryker-mutator/test-helpers": "^2.5.0", "@types/multimatch": "~4.0.0", - "stryker-mocha-framework": "^0.15.1" + "@stryker-mutator/mocha-framework": "^2.5.0" }, "peerDependencies": { "@stryker-mutator/core": "^2.0.0", diff --git a/packages/mocha-runner/src/MochaTestRunner.ts b/packages/mocha-runner/src/MochaTestRunner.ts index f90b95968a..954f64ae74 100644 --- a/packages/mocha-runner/src/MochaTestRunner.ts +++ b/packages/mocha-runner/src/MochaTestRunner.ts @@ -12,7 +12,7 @@ import { evalGlobal, mochaOptionsKey } from './utils'; const DEFAULT_TEST_PATTERN = 'test/**/*.js'; -export default class MochaTestRunner implements TestRunner { +export class MochaTestRunner implements TestRunner { private testFileNames: string[]; private readonly mochaOptions: MochaOptions; diff --git a/packages/mocha-runner/src/index.ts b/packages/mocha-runner/src/index.ts index 173c5d841f..f8457ca39f 100644 --- a/packages/mocha-runner/src/index.ts +++ b/packages/mocha-runner/src/index.ts @@ -2,7 +2,7 @@ import { BaseContext, commonTokens, declareClassPlugin, declareFactoryPlugin, In import MochaConfigEditor from './MochaConfigEditor'; import MochaOptionsLoader from './MochaOptionsLoader'; -import MochaTestRunner from './MochaTestRunner'; +import { MochaTestRunner } from './MochaTestRunner'; export const strykerPlugins = [ declareFactoryPlugin(PluginKind.ConfigEditor, 'mocha-runner', mochaConfigEditorFactory), diff --git a/packages/mocha-runner/test/integration/MochaFileResolving.spec.ts b/packages/mocha-runner/test/integration/MochaFileResolving.spec.ts index bd9b526dad..f9129765e5 100644 --- a/packages/mocha-runner/test/integration/MochaFileResolving.spec.ts +++ b/packages/mocha-runner/test/integration/MochaFileResolving.spec.ts @@ -5,7 +5,7 @@ import { expect } from 'chai'; import { testInjector } from '@stryker-mutator/test-helpers'; import MochaOptionsLoader from '../../src/MochaOptionsLoader'; -import MochaTestRunner from '../../src/MochaTestRunner'; +import { MochaTestRunner } from '../../src/MochaTestRunner'; import { mochaOptionsKey } from '../../src/utils'; describe('Mocha 6 file resolving integration', () => { diff --git a/packages/mocha-runner/test/integration/MochaFramework.it.spec.ts b/packages/mocha-runner/test/integration/MochaFramework.it.spec.ts index dfbcc69bee..7a58f7a450 100644 --- a/packages/mocha-runner/test/integration/MochaFramework.it.spec.ts +++ b/packages/mocha-runner/test/integration/MochaFramework.it.spec.ts @@ -1,11 +1,15 @@ -import { fork } from 'child_process'; +import * as path from 'path'; import { TestSelection } from '@stryker-mutator/api/test_framework'; -import { RunResult, RunStatus, TestStatus } from '@stryker-mutator/api/test_runner'; +import { TestStatus, RunStatus } from '@stryker-mutator/api/test_runner'; import { expect } from 'chai'; -import MochaTestFramework from 'stryker-mocha-framework/src/MochaTestFramework'; +import { LoggingServer, testInjector } from '@stryker-mutator/test-helpers'; +import MochaTestFramework from '@stryker-mutator/mocha-framework/src/MochaTestFramework'; +import { LogLevel } from '@stryker-mutator/api/core'; +import { commonTokens } from '@stryker-mutator/api/plugin'; -import { AUTO_START_ARGUMENT, RunMessage } from './MochaTestFrameworkIntegrationTestWorker'; +import ChildProcessProxy from '../../../core/src/child-proxy/ChildProcessProxy'; +import { MochaTestRunner } from '../../src/MochaTestRunner'; const test0: Readonly = Object.freeze({ id: 0, @@ -23,17 +27,45 @@ export function wrapInClosure(codeFragment: string) { })((Function('return this'))());`; } -describe('Integration with stryker-mocha-framework', () => { +describe('Integration with @stryker-mutator/mocha-framework', () => { let testFramework: MochaTestFramework; + let loggingServer: LoggingServer; + let sut: ChildProcessProxy; - beforeEach(() => { + beforeEach(async () => { testFramework = new MochaTestFramework(); + loggingServer = new LoggingServer(); + const resolveSampleProject: typeof path.resolve = path.resolve.bind(path, __dirname, '../../testResources/sampleProject'); + const port = await loggingServer.listen(); + testInjector.options.mochaOptions = { + file: [], + ignore: [], + spec: [resolveSampleProject('MyMathSpec.js')] + }; + testInjector.options.plugins = []; + + sut = ChildProcessProxy.create( + require.resolve('../../src/MochaTestRunner'), + { level: LogLevel.Trace, port }, + testInjector.options, + { + [commonTokens.sandboxFileNames]: [resolveSampleProject('MyMath.js'), resolveSampleProject('MyMathSpec.js')] + }, + __dirname, + MochaTestRunner + ); + await sut.proxy.init(); }); - it('should be able to select only test 0 and 3 to run', async () => { + afterEach(async () => { + await sut.dispose(); + await loggingServer.dispose(); + }); + + it('should be able to filter tests', async () => { const testHooks = wrapInClosure(testFramework.filter([test0, test3])); - const actualProcessOutput = await actRun(testHooks); - expect(actualProcessOutput.result.tests.map(test => ({ name: test.name, status: test.status }))).deep.eq([ + const actualResult = await sut.proxy.run({ testHooks }); + expect(actualResult.tests.map(test => ({ name: test.name, status: test.status }))).deep.eq([ { name: 'MyMath should be able to add two numbers', status: TestStatus.Success @@ -45,39 +77,19 @@ describe('Integration with stryker-mocha-framework', () => { ]); }); + it('should be able to clear the filter after a filtered run', async () => { + await sut.proxy.run({ testHooks: wrapInClosure(testFramework.filter([test0, test3])) }); + const actualResult = await sut.proxy.run({ testHooks: wrapInClosure(testFramework.filter([])) }); + expect(actualResult.tests).lengthOf(5); + }); + it('should be able to run beforeEach and afterEach', async () => { const testHooks = wrapInClosure( testFramework.beforeEach('console.log("beforeEach from hook");') + testFramework.afterEach('console.log("afterEach from hook");') ); - const actualProcessOutput = await actRun(testHooks); - expect(actualProcessOutput.result.status).eq(RunStatus.Complete); - expect(actualProcessOutput.stdout).includes('beforeEach from hook'); - expect(actualProcessOutput.stdout).includes('afterEach from hook'); + const actualProcessOutput = await sut.proxy.run({ testHooks }); + expect(actualProcessOutput.status).eq(RunStatus.Complete); + expect(sut.stdout).includes('beforeEach from hook'); + expect(sut.stdout).includes('afterEach from hook'); }); - - function actRun(testHooks: string | undefined): Promise<{ result: RunResult; stdout: string }> { - return new Promise<{ result: RunResult; stdout: string }>((resolve, reject) => { - const sutProxy = fork(require.resolve('./MochaTestFrameworkIntegrationTestWorker'), [AUTO_START_ARGUMENT], { - execArgv: [], - silent: true - }); - let stdout: string = ''; - sutProxy.stdout.on('data', chunk => (stdout += chunk.toString())); - const message: RunMessage = { kind: 'run', testHooks }; - sutProxy.send(message, (error: Error) => { - if (error) { - reject(error); - sutProxy.kill('SIGKILL'); - } - }); - sutProxy.on('message', (result: RunResult) => { - if (result.tests) { - resolve({ result, stdout }); - } else { - reject(result); - } - sutProxy.kill('SIGKILL'); - }); - }); - } }); diff --git a/packages/mocha-runner/test/integration/MochaTestFrameworkIntegrationTestWorker.ts b/packages/mocha-runner/test/integration/MochaTestFrameworkIntegrationTestWorker.ts deleted file mode 100644 index 9afe472dcd..0000000000 --- a/packages/mocha-runner/test/integration/MochaTestFrameworkIntegrationTestWorker.ts +++ /dev/null @@ -1,65 +0,0 @@ -import * as path from 'path'; - -import { commonTokens } from '@stryker-mutator/api/plugin'; -import { RunResult } from '@stryker-mutator/api/test_runner'; -import { testInjector } from '@stryker-mutator/test-helpers'; - -import MochaTestRunner from '../../src/MochaTestRunner'; - -export const AUTO_START_ARGUMENT = '2e164669-acf1-461c-9c05-2be139614de2'; - -export type ChildMessage = RunMessage; - -export interface RunMessage { - kind: 'run'; - testHooks?: string; -} - -export default class MochaTestFrameworkIntegrationTestWorker { - public readonly sut: MochaTestRunner; - - constructor() { - testInjector.options.mochaOptions = { - file: [], - ignore: [], - spec: [path.resolve(__dirname, '..', '..', 'testResources', 'sampleProject', 'MyMathSpec.js')] - }; - this.sut = testInjector.injector - .provideValue(commonTokens.sandboxFileNames, [ - path.resolve(__dirname, '..', '..', 'testResources', 'sampleProject', 'MyMath.js'), - path.resolve(__dirname, '..', '..', 'testResources', 'sampleProject', 'MyMathSpec.js') - ]) - .injectClass(MochaTestRunner); - - this.listenForParentProcess(); - // try { - this.sut.init(); - // } catch (err) { - // this.sendError(err); - // } - } - - public listenForParentProcess() { - process.on('message', (message: ChildMessage) => { - this.sut - .run({ testHooks: message.testHooks }) - .then(result => this.send(result)) - .catch(error => this.send(error)); - }); - } - - public send(result: RunResult) { - if (process.send) { - process.send(result); - } - } - public sendError(error: Error) { - if (process.send) { - process.send({ name: error.name, message: error.message, stack: error.stack }); - } - } -} - -if (process.argv.includes(AUTO_START_ARGUMENT)) { - new MochaTestFrameworkIntegrationTestWorker(); -} diff --git a/packages/mocha-runner/test/integration/QUnitSample.it.spec.ts b/packages/mocha-runner/test/integration/QUnitSample.it.spec.ts index 5eecd5e5aa..d8ccaa82ba 100644 --- a/packages/mocha-runner/test/integration/QUnitSample.it.spec.ts +++ b/packages/mocha-runner/test/integration/QUnitSample.it.spec.ts @@ -5,7 +5,7 @@ import { RunStatus } from '@stryker-mutator/api/test_runner'; import { testInjector } from '@stryker-mutator/test-helpers'; import { expect } from 'chai'; -import MochaTestRunner from '../../src/MochaTestRunner'; +import { MochaTestRunner } from '../../src/MochaTestRunner'; import { createMochaOptions } from '../helpers/factories'; describe('QUnit sample', () => { diff --git a/packages/mocha-runner/test/integration/SampleProject.it.spec.ts b/packages/mocha-runner/test/integration/SampleProject.it.spec.ts index 5cb57d5f41..2b7f46c24e 100644 --- a/packages/mocha-runner/test/integration/SampleProject.it.spec.ts +++ b/packages/mocha-runner/test/integration/SampleProject.it.spec.ts @@ -6,7 +6,7 @@ import { testInjector } from '@stryker-mutator/test-helpers'; import * as chai from 'chai'; import * as chaiAsPromised from 'chai-as-promised'; -import MochaTestRunner from '../../src/MochaTestRunner'; +import { MochaTestRunner } from '../../src/MochaTestRunner'; import { createMochaOptions } from '../helpers/factories'; chai.use(chaiAsPromised); diff --git a/packages/mocha-runner/test/unit/MochaTestRunner.spec.ts b/packages/mocha-runner/test/unit/MochaTestRunner.spec.ts index e937e03c64..b2398e12bd 100644 --- a/packages/mocha-runner/test/unit/MochaTestRunner.spec.ts +++ b/packages/mocha-runner/test/unit/MochaTestRunner.spec.ts @@ -10,7 +10,7 @@ import * as Mocha from 'mocha'; import sinon = require('sinon'); import LibWrapper from '../../src/LibWrapper'; import { MochaOptions } from '../../src/MochaOptions'; -import MochaTestRunner from '../../src/MochaTestRunner'; +import { MochaTestRunner } from '../../src/MochaTestRunner'; import { StrykerMochaReporter } from '../../src/StrykerMochaReporter'; import * as utils from '../../src/utils'; diff --git a/packages/mocha-runner/tsconfig.test.json b/packages/mocha-runner/tsconfig.test.json index 099a53f9b5..d4b1083f05 100644 --- a/packages/mocha-runner/tsconfig.test.json +++ b/packages/mocha-runner/tsconfig.test.json @@ -17,6 +17,9 @@ }, { "path": "../test-helpers/tsconfig.src.json" + }, + { + "path": "../core/tsconfig.src.json" } ] -} \ No newline at end of file +} diff --git a/packages/core/test/helpers/LoggingServer.ts b/packages/test-helpers/src/LoggingServer.ts similarity index 78% rename from packages/core/test/helpers/LoggingServer.ts rename to packages/test-helpers/src/LoggingServer.ts index 3edb1583d0..b0a8771266 100644 --- a/packages/core/test/helpers/LoggingServer.ts +++ b/packages/test-helpers/src/LoggingServer.ts @@ -4,13 +4,13 @@ import { parse } from 'flatted'; import * as log4js from 'log4js'; import { Observable, Subscriber } from 'rxjs'; -export default class LoggingServer { +export class LoggingServer { private readonly server: net.Server; private subscriber: Subscriber | undefined; public readonly event$: Observable; private disposed = false; - constructor(public readonly port: number) { + constructor() { this.server = net.createServer(socket => { socket.on('data', data => { // Log4js also sends "__LOG4JS__" to signal an event end. Ignore those. @@ -26,8 +26,6 @@ export default class LoggingServer { // This happens during integration tests, this is safe to ignore (log4js does that as well) }); }); - this.server.listen(this.port); - this.event$ = new Observable(subscriber => { this.subscriber = subscriber; this.server.on('close', () => { @@ -36,6 +34,19 @@ export default class LoggingServer { }); } + private alreadyListening = false; + public listen(): Promise { + if (this.alreadyListening) { + throw new Error('Server already listening'); + } else { + this.alreadyListening = true; + return new Promise(res => { + this.server.on('listening', () => res((this.server.address() as net.AddressInfo).port)); + this.server.listen(); + }); + } + } + public dispose(): Promise { if (this.disposed) { return Promise.resolve(); diff --git a/packages/test-helpers/src/index.ts b/packages/test-helpers/src/index.ts index c56bd55def..b922c3037a 100644 --- a/packages/test-helpers/src/index.ts +++ b/packages/test-helpers/src/index.ts @@ -2,3 +2,4 @@ import * as factory from './factory'; export * from './TestInjector'; export { factory }; +export * from './LoggingServer';