fix(TestRunnerOrchestrator): Error in test run (#120)

An error in a test run which was mutated now results in a killed mutant.
stryker-mutator · Jul 12, 2016 · 564f15c · 564f15c
1 parent 31e0946
commit 564f15c
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 16 deletions.
diff --git a/src/TestRunnerOrchestrator.ts b/src/TestRunnerOrchestrator.ts
@@ -115,6 +115,10 @@ export default class TestRunnerOrchestrator {
         case TestResult.Timeout:
           status = MutantStatus.TIMEDOUT;
           break;
+        case TestResult.Error:
+          log.debug('Converting a test result `error` to mutant status `killed`.');
+          status = MutantStatus.KILLED;
+          break;
         case TestResult.Complete:
           if (runResult.failed > 0) {
             status = MutantStatus.KILLED;

diff --git a/test/unit/TestRunnerOrchestratorSpec.ts b/test/unit/TestRunnerOrchestratorSpec.ts
@@ -1,10 +1,10 @@
 import TestRunnerOrchestrator from '../../src/TestRunnerOrchestrator';
 import * as sinon from 'sinon';
 import StrykerTempFolder from '../../src/utils/StrykerTempFolder';
-import {Reporter} from 'stryker-api/report';
+import {Reporter, MutantStatus, MutantResult} from 'stryker-api/report';
 import {TestSelector} from 'stryker-api/test_selector';
 import {TestRunner, RunResult, RunOptions, RunnerOptions, TestResult} from 'stryker-api/test_runner';
-import {MutantStatus, MutantResult} from 'stryker-api/report';
+import {StrykerOptions} from 'stryker-api/core';
 import IsolatedTestRunnerAdapter from '../../src/isolated-runner/IsolatedTestRunnerAdapter';
 import IsolatedTestRunnerAdapterFactory from '../../src/isolated-runner/IsolatedTestRunnerAdapterFactory';
 import * as chai from 'chai';
@@ -32,9 +32,9 @@ describe('TestRunnerOrchestrator', () => {
     { path: path.join(process.cwd(), 'aSpec.js'), shouldMutate: false },
     { path: path.join(process.cwd(), 'bSpec.js'), shouldMutate: false }
   ];
-  let strykerOptions = { testRunner: 'superRunner', port: 42 };
-  let firstTestRunner: any;
-  let secondTestRunner: any;
+  let strykerOptions: StrykerOptions = { testRunner: 'superRunner', port: 42, timeoutFactor: 1, timeoutMs: 0 };
+  let firstTestRunner: { run: sinon.SinonStub, dispose: sinon.SinonStub };
+  let secondTestRunner: { run: sinon.SinonStub, dispose: sinon.SinonStub };
   let selector: TestSelector;
   let reporter: Reporter;
 
@@ -47,11 +47,24 @@ describe('TestRunnerOrchestrator', () => {
       run: sinon.stub(),
       dispose: sinon.stub()
     };
-    firstTestRunner.run
-      .onFirstCall().returns(Promise.resolve({ result: TestResult.Complete, succeeded: 1 }))
-      .onSecondCall().returns(Promise.resolve({ result: TestResult.Complete, failed: 1 }))
-      .onThirdCall().returns(Promise.resolve({ result: TestResult.Complete }));
-    secondTestRunner.run.returns(Promise.resolve({ result: TestResult.Timeout }));
+
+    let configureTestRunner = (stub: sinon.SinonStub) => stub
+      // Run mutants
+      .withArgs({ timeout: 0 }).returns(Promise.resolve({ result: TestResult.Complete, succeeded: 1 }))
+      .withArgs({ timeout: 1 }).returns(Promise.resolve({ result: TestResult.Complete, failed: 1 }))
+      .withArgs({ timeout: 2 }).returns(Promise.resolve({ result: TestResult.Complete }))
+      .withArgs({ timeout: 3 }).returns(Promise.resolve({ result: TestResult.Timeout }))
+      .withArgs({ timeout: 4 }).returns(Promise.resolve({ result: TestResult.Error }))
+
+      // Initial test run
+      .withArgs({ timeout: 10000 })
+        .onCall(0).returns(Promise.resolve({ result: TestResult.Complete, succeeded: 1 }))
+        .onCall(1).returns(Promise.resolve({ result: TestResult.Complete, failed: 1 }))
+        .onCall(2).returns(Promise.resolve({ result: TestResult.Complete }));
+
+
+    configureTestRunner(firstTestRunner.run);
+    configureTestRunner(secondTestRunner.run);
     selector = {
       select: sinon.stub().returns('some selected contents')
     };
@@ -135,7 +148,7 @@ describe('TestRunnerOrchestrator', () => {
       });
     });
 
-    describe('runMutations() with 2 cpus and 4 mutants', () => {
+    describe('runMutations() with 2 cpus and 5 mutants', () => {
       let donePromise: Promise<void>;
       let mutants: any[];
       let mutantResults: MutantResult[];
@@ -146,7 +159,7 @@ describe('TestRunnerOrchestrator', () => {
         var untestedMutant = mockMutant(0);
         untestedMutant.scopedTestIds = [];
 
-        mutants = [untestedMutant, mockMutant(1), mockMutant(2), mockMutant(3)];
+        mutants = [untestedMutant, mockMutant(1), mockMutant(2), mockMutant(3), mockMutant(4)];
         return sut.runMutations(mutants)
           .then(results => mutantResults = results);
       });
@@ -170,31 +183,33 @@ describe('TestRunnerOrchestrator', () => {
         expect(firstTestRunner.run).to.have.been.calledTwice;
       });
 
-      it('should have ran mutant 2 on the second test runner', () => {
-        expect(secondTestRunner.run).to.have.been.calledOnce;
+      it('should have ran mutant 2 and mutant 4 on the second test runner', () => {
+        expect(secondTestRunner.run).to.have.been.calledTwice;
       });
 
       it('should have reported onMutantTested on all mutants', () => {
-        expect(reporter.onMutantTested).to.have.callCount(4)
+        expect(reporter.onMutantTested).to.have.callCount(5)
         expect(reporter.onMutantTested).to.have.been.calledWith(mutantResults[0]);
         expect(reporter.onMutantTested).to.have.been.calledWith(mutantResults[1]);
         expect(reporter.onMutantTested).to.have.been.calledWith(mutantResults[2]);
         expect(reporter.onMutantTested).to.have.been.calledWith(mutantResults[3]);
+        expect(reporter.onMutantTested).to.have.been.calledWith(mutantResults[4]);
       });
 
       it('should have reported onAllMutantsTested', () => {
         expect(reporter.onAllMutantsTested).to.have.been.calledWith(mutantResults);
       });
 
       it('should eventually resolve the correct mutant results', () => {
-        expect(mutantResults.length).to.be.eq(4);
+        expect(mutantResults.length).to.be.eq(5);
 
         let sortedMutantResults = _.sortBy(mutantResults, r => r.sourceFilePath);
 
         expect(sortedMutantResults[0].status).to.be.eq(MutantStatus.UNTESTED);
         expect(sortedMutantResults[1].status).to.be.eq(MutantStatus.KILLED);
         expect(sortedMutantResults[2].status).to.be.eq(MutantStatus.SURVIVED);
         expect(sortedMutantResults[3].status).to.be.eq(MutantStatus.TIMEDOUT);
+        expect(sortedMutantResults[4].status).to.be.eq(MutantStatus.KILLED);
       });
     });