From 564f15c35d65876cf2a0aafe0e40b1c7c94359ee Mon Sep 17 00:00:00 2001 From: Nico Jansen Date: Tue, 12 Jul 2016 18:29:09 +0200 Subject: [PATCH] fix(TestRunnerOrchestrator): Error in test run (#120) An error in a test run which was mutated now results in a killed mutant. --- src/TestRunnerOrchestrator.ts | 4 +++ test/unit/TestRunnerOrchestratorSpec.ts | 47 ++++++++++++++++--------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/TestRunnerOrchestrator.ts b/src/TestRunnerOrchestrator.ts index 8a752a3252..a5add2df7c 100644 --- a/src/TestRunnerOrchestrator.ts +++ b/src/TestRunnerOrchestrator.ts @@ -115,6 +115,10 @@ export default class TestRunnerOrchestrator { case TestResult.Timeout: status = MutantStatus.TIMEDOUT; break; + case TestResult.Error: + log.debug('Converting a test result `error` to mutant status `killed`.'); + status = MutantStatus.KILLED; + break; case TestResult.Complete: if (runResult.failed > 0) { status = MutantStatus.KILLED; diff --git a/test/unit/TestRunnerOrchestratorSpec.ts b/test/unit/TestRunnerOrchestratorSpec.ts index 4ed7c04a77..af2d1e9612 100644 --- a/test/unit/TestRunnerOrchestratorSpec.ts +++ b/test/unit/TestRunnerOrchestratorSpec.ts @@ -1,10 +1,10 @@ import TestRunnerOrchestrator from '../../src/TestRunnerOrchestrator'; import * as sinon from 'sinon'; import StrykerTempFolder from '../../src/utils/StrykerTempFolder'; -import {Reporter} from 'stryker-api/report'; +import {Reporter, MutantStatus, MutantResult} from 'stryker-api/report'; import {TestSelector} from 'stryker-api/test_selector'; import {TestRunner, RunResult, RunOptions, RunnerOptions, TestResult} from 'stryker-api/test_runner'; -import {MutantStatus, MutantResult} from 'stryker-api/report'; +import {StrykerOptions} from 'stryker-api/core'; import IsolatedTestRunnerAdapter from '../../src/isolated-runner/IsolatedTestRunnerAdapter'; import IsolatedTestRunnerAdapterFactory from '../../src/isolated-runner/IsolatedTestRunnerAdapterFactory'; import * as chai from 'chai'; @@ -32,9 +32,9 @@ describe('TestRunnerOrchestrator', () => { { path: path.join(process.cwd(), 'aSpec.js'), shouldMutate: false }, { path: path.join(process.cwd(), 'bSpec.js'), shouldMutate: false } ]; - let strykerOptions = { testRunner: 'superRunner', port: 42 }; - let firstTestRunner: any; - let secondTestRunner: any; + let strykerOptions: StrykerOptions = { testRunner: 'superRunner', port: 42, timeoutFactor: 1, timeoutMs: 0 }; + let firstTestRunner: { run: sinon.SinonStub, dispose: sinon.SinonStub }; + let secondTestRunner: { run: sinon.SinonStub, dispose: sinon.SinonStub }; let selector: TestSelector; let reporter: Reporter; @@ -47,11 +47,24 @@ describe('TestRunnerOrchestrator', () => { run: sinon.stub(), dispose: sinon.stub() }; - firstTestRunner.run - .onFirstCall().returns(Promise.resolve({ result: TestResult.Complete, succeeded: 1 })) - .onSecondCall().returns(Promise.resolve({ result: TestResult.Complete, failed: 1 })) - .onThirdCall().returns(Promise.resolve({ result: TestResult.Complete })); - secondTestRunner.run.returns(Promise.resolve({ result: TestResult.Timeout })); + + let configureTestRunner = (stub: sinon.SinonStub) => stub + // Run mutants + .withArgs({ timeout: 0 }).returns(Promise.resolve({ result: TestResult.Complete, succeeded: 1 })) + .withArgs({ timeout: 1 }).returns(Promise.resolve({ result: TestResult.Complete, failed: 1 })) + .withArgs({ timeout: 2 }).returns(Promise.resolve({ result: TestResult.Complete })) + .withArgs({ timeout: 3 }).returns(Promise.resolve({ result: TestResult.Timeout })) + .withArgs({ timeout: 4 }).returns(Promise.resolve({ result: TestResult.Error })) + + // Initial test run + .withArgs({ timeout: 10000 }) + .onCall(0).returns(Promise.resolve({ result: TestResult.Complete, succeeded: 1 })) + .onCall(1).returns(Promise.resolve({ result: TestResult.Complete, failed: 1 })) + .onCall(2).returns(Promise.resolve({ result: TestResult.Complete })); + + + configureTestRunner(firstTestRunner.run); + configureTestRunner(secondTestRunner.run); selector = { select: sinon.stub().returns('some selected contents') }; @@ -135,7 +148,7 @@ describe('TestRunnerOrchestrator', () => { }); }); - describe('runMutations() with 2 cpus and 4 mutants', () => { + describe('runMutations() with 2 cpus and 5 mutants', () => { let donePromise: Promise; let mutants: any[]; let mutantResults: MutantResult[]; @@ -146,7 +159,7 @@ describe('TestRunnerOrchestrator', () => { var untestedMutant = mockMutant(0); untestedMutant.scopedTestIds = []; - mutants = [untestedMutant, mockMutant(1), mockMutant(2), mockMutant(3)]; + mutants = [untestedMutant, mockMutant(1), mockMutant(2), mockMutant(3), mockMutant(4)]; return sut.runMutations(mutants) .then(results => mutantResults = results); }); @@ -170,16 +183,17 @@ describe('TestRunnerOrchestrator', () => { expect(firstTestRunner.run).to.have.been.calledTwice; }); - it('should have ran mutant 2 on the second test runner', () => { - expect(secondTestRunner.run).to.have.been.calledOnce; + it('should have ran mutant 2 and mutant 4 on the second test runner', () => { + expect(secondTestRunner.run).to.have.been.calledTwice; }); it('should have reported onMutantTested on all mutants', () => { - expect(reporter.onMutantTested).to.have.callCount(4) + expect(reporter.onMutantTested).to.have.callCount(5) expect(reporter.onMutantTested).to.have.been.calledWith(mutantResults[0]); expect(reporter.onMutantTested).to.have.been.calledWith(mutantResults[1]); expect(reporter.onMutantTested).to.have.been.calledWith(mutantResults[2]); expect(reporter.onMutantTested).to.have.been.calledWith(mutantResults[3]); + expect(reporter.onMutantTested).to.have.been.calledWith(mutantResults[4]); }); it('should have reported onAllMutantsTested', () => { @@ -187,7 +201,7 @@ describe('TestRunnerOrchestrator', () => { }); it('should eventually resolve the correct mutant results', () => { - expect(mutantResults.length).to.be.eq(4); + expect(mutantResults.length).to.be.eq(5); let sortedMutantResults = _.sortBy(mutantResults, r => r.sourceFilePath); @@ -195,6 +209,7 @@ describe('TestRunnerOrchestrator', () => { expect(sortedMutantResults[1].status).to.be.eq(MutantStatus.KILLED); expect(sortedMutantResults[2].status).to.be.eq(MutantStatus.SURVIVED); expect(sortedMutantResults[3].status).to.be.eq(MutantStatus.TIMEDOUT); + expect(sortedMutantResults[4].status).to.be.eq(MutantStatus.KILLED); }); });