diff --git a/package.json b/package.json index b9a1c6e44d..1435f4d1c7 100644 --- a/package.json +++ b/package.json @@ -69,12 +69,12 @@ "mocha-sinon": "^1.1.4", "sinon": "^1.17.2", "sinon-chai": "^2.8.0", - "stryker-api": "^0.1.0", + "stryker-api": "^0.1.1", "stryker-karma-runner": "^0.1.0", "typescript": "^1.8.9", "typings": "^0.7.11" }, "peerDependencies": { - "stryker-api": "^0.1.0" + "stryker-api": "^0.1.1" } } diff --git a/src/TestRunnerOrchestrator.ts b/src/TestRunnerOrchestrator.ts index a5add2df7c..4b5e838c47 100644 --- a/src/TestRunnerOrchestrator.ts +++ b/src/TestRunnerOrchestrator.ts @@ -1,9 +1,9 @@ import {StrykerOptions, InputFile} from 'stryker-api/core'; -import {RunResult, RunnerOptions, TestResult} from 'stryker-api/test_runner'; +import {TestRunner, RunResult, RunnerOptions, TestResult} from 'stryker-api/test_runner'; import {TestSelector} from 'stryker-api/test_selector'; import StrykerTempFolder from './utils/StrykerTempFolder'; -import IsolatedTestRunnerAdapter from './isolated-runner/IsolatedTestRunnerAdapter'; import IsolatedTestRunnerAdapterFactory from './isolated-runner/IsolatedTestRunnerAdapterFactory'; +import IsolatedTestRunnerAdapter from './isolated-runner/IsolatedTestRunnerAdapter'; import * as path from 'path'; import * as os from 'os'; import * as _ from 'lodash'; @@ -21,7 +21,7 @@ interface FileMap { interface TestRunnerSandbox { index: number; - runnerAdapter: IsolatedTestRunnerAdapter; + runner: IsolatedTestRunnerAdapter; fileMap: FileMap; testSelectionFilePath: string; } @@ -49,17 +49,14 @@ export default class TestRunnerOrchestrator { private initialRunWithTestSelector() { let testSelectionFilePath = this.createTestSelectorFileName(this.createTempFolder()); - let runnerAdapter = this.createTestRunner(this.files, true, testSelectionFilePath); + let runner = this.createTestRunner(this.files, true, testSelectionFilePath); let sandbox: TestRunnerSandbox = { - runnerAdapter, + runner, fileMap: null, testSelectionFilePath, index: 0 }; - return this.runSingleTestsRecursive(sandbox, [], 0).then((testResults) => { - runnerAdapter.dispose(); - return testResults; - }); + return this.runSingleTestsRecursive(sandbox, [], 0).then((testResults) => runner.dispose().then(() => testResults)); } runMutations(mutants: Mutant[]): Promise { @@ -75,7 +72,7 @@ export default class TestRunnerOrchestrator { let sandbox = sandboxes.pop(); let sourceFileCopy = sandbox.fileMap[mutant.filename]; return Promise.all([mutant.save(sourceFileCopy), this.selectTestsIfPossible(sandbox, mutant.scopedTestIds)]) - .then(() => sandbox.runnerAdapter.run({ timeout: this.calculateTimeout(mutant.timeSpentScopedTests) })) + .then(() => sandbox.runner.run({ timeout: this.calculateTimeout(mutant.timeSpentScopedTests) })) .then((runResult) => { let result = this.collectFrozenMutantResult(mutant, runResult); results.push(result); @@ -92,8 +89,8 @@ export default class TestRunnerOrchestrator { } return new PromisePool(promiseProducer, sandboxes.length) .start() - .then(() => sandboxes.forEach(testRunner => testRunner.runnerAdapter.dispose())) .then(() => this.reportAllMutantsTested(results)) + .then(() => Promise.all(sandboxes.map(testRunner => testRunner.runner.dispose()))) .then(() => results); }); } @@ -153,7 +150,7 @@ export default class TestRunnerOrchestrator { return new Promise(resolve => { this.selectTestsIfPossible(sandbox, [currentTestIndex]) - .then(() => sandbox.runnerAdapter.run({ timeout: 10000 })) + .then(() => sandbox.runner.run({ timeout: 10000 })) .then(runResult => { if (runResult.result === TestResult.Complete && runResult.succeeded > 0 || runResult.failed > 0) { runResults[currentTestIndex] = runResult; @@ -163,7 +160,7 @@ export default class TestRunnerOrchestrator { // If this was iteration n+1 (n = number of tests), the runResult.result will be Complete, so we don't record it runResults[currentTestIndex] = runResult; } - sandbox.runnerAdapter.dispose(); + sandbox.runner.dispose(); resolve(runResults); } }); @@ -176,35 +173,37 @@ export default class TestRunnerOrchestrator { let allPromises: Promise[] = []; log.info(`Creating ${cpuCount} test runners (based on cpu count)`); for (let i = 0; i < cpuCount; i++) { - allPromises.push(this.createSandbox(i).then(sandbox => testRunnerSandboxes.push(sandbox))); + allPromises.push(this.createInitializedSandbox(i)); } - return Promise.all(allPromises).then(() => testRunnerSandboxes); + return Promise.all(allPromises); + } private selectTestsIfPossible(sandbox: TestRunnerSandbox, ids: number[]): Promise { if (this.testSelector) { let fileContent = this.testSelector.select(ids); return StrykerTempFolder.writeFile(sandbox.testSelectionFilePath, fileContent); - }else{ + } else { return Promise.resolve(void 0); } } - private createSandbox(index: number): Promise { + private createInitializedSandbox(index: number): Promise { var tempFolder = this.createTempFolder(); return this.copyAllFilesToFolder(tempFolder).then(fileMap => { let runnerFiles: InputFile[] = []; let testSelectionFilePath: string = null; - if(this.testSelector){ + if (this.testSelector) { testSelectionFilePath = this.createTestSelectorFileName(tempFolder); } this.files.forEach(originalFile => runnerFiles.push({ path: fileMap[originalFile.path], shouldMutate: originalFile.shouldMutate })); - return { + let runner = this.createTestRunner(runnerFiles, false, testSelectionFilePath, index); + return runner.init().then(() => ({ index, fileMap, - runnerAdapter: this.createTestRunner(runnerFiles, false, testSelectionFilePath, index), + runner, testSelectionFilePath - }; + })); }); } diff --git a/src/isolated-runner/IsolatedTestRunnerAdapter.ts b/src/isolated-runner/IsolatedTestRunnerAdapter.ts index eaf3595d02..598a314c49 100644 --- a/src/isolated-runner/IsolatedTestRunnerAdapter.ts +++ b/src/isolated-runner/IsolatedTestRunnerAdapter.ts @@ -1,7 +1,7 @@ import {TestRunner, RunResult, RunOptions, RunnerOptions, TestResult} from 'stryker-api/test_runner'; import {StrykerOptions} from 'stryker-api/core'; import {fork, ChildProcess} from 'child_process'; -import Message, {MessageType} from './Message'; +import Message, {MessageType } from './Message'; import StartMessageBody from './StartMessageBody'; import RunMessageBody from './RunMessageBody'; import ResultMessageBody from './ResultMessageBody'; @@ -9,6 +9,7 @@ import * as _ from 'lodash'; import * as log4js from 'log4js'; const log = log4js.getLogger('IsolatedTestRunnerAdapter'); +const MAX_WAIT_FOR_DISPOSE = 2000; /** * Runs the given test runner in a child process and forwards reports about test results @@ -17,10 +18,15 @@ const log = log4js.getLogger('IsolatedTestRunnerAdapter'); export default class TestRunnerChildProcessAdapter implements TestRunner { private workerProcess: ChildProcess; - private currentPromiseFulfillmentCallback: (result: RunResult) => void; - private currentPromise: Promise; + private initPromiseFulfillmentCallback: () => void; + private runPromiseFulfillmentCallback: (result: RunResult) => void; + private disposePromiseFulfillmentCallback: () => void; + private initPromise: Promise; + private runPromise: Promise; + private disposingPromise: Promise; private currentTimeoutTimer: NodeJS.Timer; private currentRunStartedTimestamp: Date; + private isDisposing: boolean; constructor(private realTestRunnerName: string, private options: RunnerOptions) { this.startWorker(); @@ -56,7 +62,15 @@ export default class TestRunnerChildProcessAdapter implements TestRunner { this.clearCurrentTimer(); switch (message.type) { case MessageType.Result: - this.handleResultMessage(message); + if (!this.isDisposing) { + this.handleResultMessage(message); + } + break; + case MessageType.InitDone: + this.initPromiseFulfillmentCallback(); + break; + case MessageType.DisposeDone: + this.disposePromiseFulfillmentCallback(); break; default: log.error(`Retrieved unrecognized message from child process: ${JSON.stringify(message)}`) @@ -65,22 +79,41 @@ export default class TestRunnerChildProcessAdapter implements TestRunner { }); } + init(): Promise { + this.initPromise = new Promise(resolve => this.initPromiseFulfillmentCallback = resolve); + this.sendInitCommand(); + return this.initPromise; + } + run(options: RunOptions): Promise { this.clearCurrentTimer(); if (options.timeout) { this.markNoResultTimeout(options.timeout); } - this.currentPromise = new Promise(resolve => { - this.currentPromiseFulfillmentCallback = resolve; + this.runPromise = new Promise(resolve => { + this.runPromiseFulfillmentCallback = resolve; this.sendRunCommand(options); this.currentRunStartedTimestamp = new Date(); }); - return this.currentPromise; + return this.runPromise; } - dispose() { - this.clearCurrentTimer(); - this.workerProcess.kill(); + dispose(): Promise { + if (this.isDisposing) { + return this.disposingPromise; + } else { + this.isDisposing = true; + this.disposingPromise = new Promise(resolve => this.disposePromiseFulfillmentCallback = resolve) + .then(() => { + clearTimeout(timer); + this.workerProcess.kill(); + this.isDisposing = false; + }); + this.clearCurrentTimer(); + this.sendDisposeCommand(); + let timer = setTimeout(this.disposePromiseFulfillmentCallback, MAX_WAIT_FOR_DISPOSE); + return this.disposingPromise; + } } private sendRunCommand(options: RunOptions) { @@ -104,9 +137,17 @@ export default class TestRunnerChildProcessAdapter implements TestRunner { this.workerProcess.send(startMessage); } + private sendInitCommand() { + this.workerProcess.send(this.emptyMessage(MessageType.Init)); + } + + private sendDisposeCommand() { + this.workerProcess.send(this.emptyMessage(MessageType.Dispose)); + } + private handleResultMessage(message: Message) { message.body.result.timeSpent = (new Date().getTime() - this.currentRunStartedTimestamp.getTime()); - this.currentPromiseFulfillmentCallback(message.body.result); + this.runPromiseFulfillmentCallback(message.body.result); } private clearCurrentTimer() { @@ -122,8 +163,13 @@ export default class TestRunnerChildProcessAdapter implements TestRunner { } private handleTimeout() { - this.workerProcess.kill(); - this.startWorker(); - this.currentPromiseFulfillmentCallback({ result: TestResult.Timeout }); + this.dispose() + .then(() => this.startWorker()) + .then(() => this.init()) + .then(() => this.runPromiseFulfillmentCallback({ result: TestResult.Timeout })) + } + + private emptyMessage(type: MessageType): Message { + return { type }; } } \ No newline at end of file diff --git a/src/isolated-runner/IsolatedTestRunnerAdapterWorker.ts b/src/isolated-runner/IsolatedTestRunnerAdapterWorker.ts index 39f8074d18..4f933a9822 100644 --- a/src/isolated-runner/IsolatedTestRunnerAdapterWorker.ts +++ b/src/isolated-runner/IsolatedTestRunnerAdapterWorker.ts @@ -4,10 +4,14 @@ import StartMessageBody from './StartMessageBody'; import RunMessageBody from './RunMessageBody'; import ResultMessageBody from './ResultMessageBody'; import PluginLoader from '../PluginLoader'; +import * as log4js from 'log4js'; +import {isPromise} from '../utils/objectUtils'; + +const log = log4js.getLogger('TestRunnerChildProcessAdapterWorker'); class TestRunnerChildProcessAdapterWorker { - underlyingTestRunner: TestRunner; + private underlyingTestRunner: TestRunner; constructor() { this.listenToMessages(); @@ -22,6 +26,14 @@ class TestRunnerChildProcessAdapterWorker { case MessageType.Run: this.run(message.body); break; + case MessageType.Init: + this.init(); + break; + case MessageType.Dispose: + this.dispose(); + break; + default: + log.warn('Received unsupported message: {}', JSON.stringify(message)); } }); } @@ -31,6 +43,38 @@ class TestRunnerChildProcessAdapterWorker { this.underlyingTestRunner = TestRunnerFactory.instance().create(body.runnerName, body.runnerOptions); } + init() { + let initPromise: Promise | void = void 0; + if (this.underlyingTestRunner.init) { + initPromise = this.underlyingTestRunner.init(); + } + if (isPromise(initPromise)) { + initPromise.then(this.sendInitDone); + } else { + this.sendInitDone(); + } + } + + sendInitDone() { + process.send({ type: MessageType.InitDone }); + } + + dispose() { + let disposePromise: Promise | void = void 0; + if (this.underlyingTestRunner.dispose) { + disposePromise = this.underlyingTestRunner.dispose(); + } + if (isPromise(disposePromise)) { + disposePromise.then(this.sendDisposeDone); + } else { + this.sendDisposeDone(); + } + } + + sendDisposeDone() { + process.send({ type: MessageType.DisposeDone }); + } + run(body: RunMessageBody) { this.underlyingTestRunner.run(body.runOptions).then(this.reportResult, this.reportErrorResult); } diff --git a/src/isolated-runner/Message.ts b/src/isolated-runner/Message.ts index 9c3b835f63..abddb1e551 100644 --- a/src/isolated-runner/Message.ts +++ b/src/isolated-runner/Message.ts @@ -1,13 +1,17 @@ -export enum MessageType{ +export enum MessageType { Start, + Init, + InitDone, Run, - Result + Result, + Dispose, + DisposeDone } -interface Message { +export interface Message { type: MessageType; - body: T; + body?: T; } export default Message; \ No newline at end of file diff --git a/test/helpers/log4jsMock.ts b/test/helpers/log4jsMock.ts index bc988680ef..939a6f7b77 100644 --- a/test/helpers/log4jsMock.ts +++ b/test/helpers/log4jsMock.ts @@ -1,4 +1,3 @@ -console.log('l4js:', require.resolve('log4js')); import * as log4js from 'log4js'; import * as sinon from 'sinon'; diff --git a/test/integration/isolated-runner/IsolatedTestRunnerAdapterSpec.ts b/test/integration/isolated-runner/IsolatedTestRunnerAdapterSpec.ts index 51a8843c78..a99ab8085a 100644 --- a/test/integration/isolated-runner/IsolatedTestRunnerAdapterSpec.ts +++ b/test/integration/isolated-runner/IsolatedTestRunnerAdapterSpec.ts @@ -5,18 +5,22 @@ import * as chai from 'chai'; import * as chaiAsPromised from 'chai-as-promised'; chai.use(chaiAsPromised); let expect = chai.expect; +import * as log4js from 'log4js'; -describe('TestRunnerChildProcessAdapter', function() { +describe('TestRunnerChildProcessAdapter', function () { this.timeout(10000); - + let sut: TestRunnerChildProcessAdapter; let options: RunnerOptions = { strykerOptions: { - plugins: ['../../test/integration/isolated-runner/DirectResolvedTestRunner', '../../test/integration/isolated-runner/NeverResolvedTestRunner'], + plugins: [ + '../../test/integration/isolated-runner/DirectResolvedTestRunner', + '../../test/integration/isolated-runner/NeverResolvedTestRunner', + '../../test/integration/isolated-runner/SlowInitAndDisposeTestRunner'], testRunner: 'karma', testFrameork: 'jasmine', - port: null + port: null }, files: [], port: null, @@ -35,6 +39,7 @@ describe('TestRunnerChildProcessAdapter', function() { describe('when test runner behind never responds', () => { before(() => { sut = new TestRunnerChildProcessAdapter('never-resolved', options); + return sut.init(); }); it('should run and resolve in a timeout', () => @@ -44,4 +49,18 @@ describe('TestRunnerChildProcessAdapter', function() { expect(sut.run({ timeout: 1000 }).then(() => sut.run({ timeout: 1000 }))).to.eventually.satisfy((result: RunResult) => result.result === TestResult.Timeout)); }); + describe('when test runner behind has a slow init and dispose cycle', () => { + before(() => { + sut = new TestRunnerChildProcessAdapter('slow-init-dispose', options); + return sut.init(); + }); + it('should run only after it is initialized', + () => expect(sut.run({ timeout: 20 })).to.eventually.satisfy((result: RunResult) => result.result === TestResult.Complete)); + + it('should be able to run twice in quick succession', + () => expect(sut.run({ timeout: 20 }).then(() => sut.run({ timeout: 20 }))).to.eventually.satisfy((result: RunResult) => result.result === TestResult.Complete)); + + after(() => sut.dispose()); + }); + }); \ No newline at end of file diff --git a/test/integration/isolated-runner/SlowInitAndDisposeTestRunner.ts b/test/integration/isolated-runner/SlowInitAndDisposeTestRunner.ts new file mode 100644 index 0000000000..695fa1a26f --- /dev/null +++ b/test/integration/isolated-runner/SlowInitAndDisposeTestRunner.ts @@ -0,0 +1,30 @@ +import {TestRunnerFactory, TestRunner, RunOptions, RunResult, TestResult} from 'stryker-api/test_runner'; + +class SlowInitAndDisposeTestRunner implements TestRunner { + + inInit: boolean; + runResult: RunResult = { result: TestResult.Complete, testNames: []}; + + init(){ + return new Promise( resolve => { + this.inInit = true; + setTimeout(() => { + this.inInit = false; + resolve(); + }, 1000); + }); + } + + run(options: RunOptions){ + if(this.inInit){ + throw new Error("Test should fail! Not yet initialized!") + } + return new Promise(res => res(this.runResult)); + } + + dispose(){ + return this.init(); + } +} + +TestRunnerFactory.instance().register('slow-init-dispose', SlowInitAndDisposeTestRunner); \ No newline at end of file diff --git a/test/unit/TestRunnerOrchestratorSpec.ts b/test/unit/TestRunnerOrchestratorSpec.ts index af2d1e9612..e2a6f3b8b0 100644 --- a/test/unit/TestRunnerOrchestratorSpec.ts +++ b/test/unit/TestRunnerOrchestratorSpec.ts @@ -40,12 +40,14 @@ describe('TestRunnerOrchestrator', () => { beforeEach(() => { firstTestRunner = { + init: sinon.stub().returns(Promise.resolve()), run: sinon.stub(), - dispose: sinon.stub() + dispose: sinon.stub().returns(Promise.resolve()) }; secondTestRunner = { + init: sinon.stub().returns(Promise.resolve()), run: sinon.stub(), - dispose: sinon.stub() + dispose: sinon.stub().returns(Promise.resolve()) }; let configureTestRunner = (stub: sinon.SinonStub) => stub diff --git a/test/unit/isolated-runner/IsolatedTestRunnerAdapterSpec.ts b/test/unit/isolated-runner/IsolatedTestRunnerAdapterSpec.ts index 14caf20e79..142c632a3e 100644 --- a/test/unit/isolated-runner/IsolatedTestRunnerAdapterSpec.ts +++ b/test/unit/isolated-runner/IsolatedTestRunnerAdapterSpec.ts @@ -65,38 +65,58 @@ describe('IsolatedTestRunnerAdapter', () => { }); describe('and a timeout occurred', () => { + beforeEach(() => { clock.tick(2100); }); - it('should resolve the promise in a Timeout', () => { - return expect(runPromise).to.eventually.satisfy((result: RunResult) => result.result === TestResult.Timeout); + it('should send `dispose` to worker process', () => expect(fakeChildProcess.send).to.have.been.calledWith({ type: MessageType.Dispose })); + + let actAssertTimeout = () => { + it('should kill the child process and start a new one', () => { + expect(fakeChildProcess.kill).to.have.been.calledWith(); + expect(child_process.fork).to.have.been.called.callCount(2); + }); + + describe('and to init', () => { + let actualRunResult: RunResult; + beforeEach(() => { + receiveMessage({ type: MessageType.InitDone }); + }); + + it('should result in a `timeout` after the restart', () => expect(runPromise).to.eventually.satisfy((result: RunResult) => result.result === TestResult.Timeout)); + }); + }; + + describe('and child process responses to dispose', () => { + beforeEach(() => { + receiveMessage({ type: MessageType.DisposeDone }); + return sut.dispose(); // should return newly created promise + }); + + actAssertTimeout(); }); - it('should kill the child process and start a new one', () => { - expect(fakeChildProcess.kill).to.have.been.calledWith(); - expect(child_process.fork).to.have.been.called.callCount(2); + describe('and child process is unresponsive', () => { + beforeEach(() => { + clock.tick(2100); // default wait for child process is 2000 + return sut.dispose(); // should return newly created promise + }); + + actAssertTimeout(); }); + }); describe('and a result message occurred after 1900 ms', () => { - function receiveResultMessage() { - let callback: (message: Message) => void = fakeChildProcess.on.getCall(0).args[1]; - let message = { type: MessageType.Result, body: { result: { result: TestResult.Complete } } }; - callback(message); - return message; - } - let expectedMessage: Message; beforeEach(() => { clock.tick(1900); expectedMessage = receiveResultMessage(); }); - it('should pass along the result', () => { - return expect(runPromise).to.eventually.eq(expectedMessage.body.result); - }); + it('should pass along the result', () => expect(runPromise).to.eventually.eq(expectedMessage.body.result)); describe('when we run a second time, wait 500ms and then receive the second result', () => { @@ -115,6 +135,18 @@ describe('IsolatedTestRunnerAdapter', () => { }); }); + let receiveResultMessage = () => { + let message = { type: MessageType.Result, body: { result: { result: TestResult.Complete } } }; + receiveMessage(message); + return message; + }; + + let receiveMessage = (message: Message) => { + let callback: (message: Message) => void = fakeChildProcess.on.getCall(0).args[1]; + callback(message); + return message; + }; + afterEach(() => { clock.restore(); sinonSandbox.restore();