fix(reporters): correctly report avg tests/mutants (#2458)

Correctly report the average tests/mutant metric in the clear-text reporter

e2e/test/mocha-javascript/verify/verify.ts

@@ -25,10 +25,4 @@ describe('Verify stryker has ran correctly', () => {
     });
   });
 
-  it('should report html files', () => {
-    expectExists('reports/mutation/html/index.html');
-    expectExists('reports/mutation/html/mutation-test-elements.js');
-    expectExists('reports/mutation/html/stryker-80x80.png');
-    expectExists('reports/mutation/html/bind-mutation-test-report.js');
-  });
 });

e2e/test/reporters-e2e/.mocharc.jsonc

@@ -0,0 +1,4 @@
+{
+    "require": "./test/helpers/testSetup.js",
+    "spec": ["test/unit/*.js"]
+}

e2e/test/reporters-e2e/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "reporters-e2e",
+  "version": "0.0.0",
+  "private": true,
+  "description": "A module to perform an integration test",
+  "main": "index.js",
+  "scripts": {
+    "pretest": "rimraf \"reports\"",
+    "test": "mkdir -p reports && stryker run > reports/stdout.txt",
+    "test:unit": "mocha",
+    "posttest": "mocha --no-config --require ../../tasks/ts-node-register.js verify/*.ts"
+  },
+  "author": "",
+  "license": "ISC"
+}

e2e/test/reporters-e2e/src/Add.js

@@ -0,0 +1,26 @@
+module.exports.add = function(num1, num2) {
+  return num1 + num2;
+};
+
+module.exports.addOne = function(number) {
+  number++;
+  return number;
+};
+
+module.exports.negate = function(number) {
+  return -number;
+};
+
+module.exports.notCovered = function(number) {
+  return number > 10;
+};
+
+module.exports.isNegativeNumber = function(number) {
+  var isNegative = false;
+  if(number < 0){
+    isNegative = true;
+  }
+  return isNegative;
+};
+
+

e2e/test/reporters-e2e/src/Circle.js

@@ -0,0 +1,8 @@
+module.exports.getCircumference = function(radius) {
+  //Function to test multiple math mutations in a single function.
+  return 2 * Math.PI * radius;
+};
+
+module.exports.untestedFunction = function() {
+  var i = 5 / 2 * 3;
+};

e2e/test/reporters-e2e/stryker.conf.json

@@ -0,0 +1,11 @@
+{
+  "$schema": "../../node_modules/@stryker-mutator/core/schema/stryker-schema.json",
+  "testRunner": "mocha",
+  "concurrency": 2,
+  "coverageAnalysis": "perTest",
+  "reporters": ["clear-text", "html", "event-recorder"],
+  "plugins": [
+    "@stryker-mutator/mocha-runner"
+  ],
+  "allowConsoleColors": false
+}

e2e/test/reporters-e2e/test/helpers/testSetup.js

@@ -0,0 +1,5 @@
+exports.mochaHooks = {
+  beforeAll() {
+    global.expect = require('chai').expect;
+  }
+}

e2e/test/reporters-e2e/test/unit/AddSpec.js

@@ -0,0 +1,52 @@
+var addModule = require('../../src/Add');
+var add = addModule.add;
+var addOne = addModule.addOne;
+var isNegativeNumber = addModule.isNegativeNumber;
+var negate = addModule.negate;
+var notCovered = addModule.notCovered;
+
+describe('Add', function() {
+  it('should be able to add two numbers', function() {
+    var num1 = 2;
+    var num2 = 5;
+    var expected = num1 + num2;
+
+    var actual = add(num1, num2);
+
+    expect(actual).to.be.equal(expected);
+  });
+
+  it('should be able 1 to a number', function() {
+    var number = 2;
+    var expected = 3;
+
+    var actual = addOne(number);
+
+    expect(actual).to.be.equal(expected);
+  });
+
+  it('should be able negate a number', function() {
+    var number = 2;
+    var expected = -2;
+
+    var actual = negate(number);
+
+    expect(actual).to.be.equal(expected);
+  });
+
+  it('should be able to recognize a negative number', function() {
+    var number = -2;
+
+    var isNegative = isNegativeNumber(number);
+
+    expect(isNegative).to.be.true;
+  });
+
+  it('should be able to recognize that 0 is not a negative number', function() {
+    var number = 0;
+
+    var isNegative = isNegativeNumber(number);
+
+    expect(isNegative).to.be.false;
+  });
+});

e2e/test/reporters-e2e/test/unit/CircleSpec.js

@@ -0,0 +1,13 @@
+var circleModule = require('../../src/Circle');
+var getCircumference = circleModule.getCircumference;
+
+describe('Circle', function() {
+  it('should have a circumference of 2PI when the radius is 1', function() {
+    var radius = 1;
+    var expectedCircumference = 2 * Math.PI;
+
+    var circumference = getCircumference(radius);
+
+    expect(circumference).to.be.equal(expectedCircumference);
+  });
+});

e2e/test/reporters-e2e/verify/verify.ts

@@ -0,0 +1,61 @@
+import { expect } from 'chai';
+import * as fs from 'fs';
+import { describe } from 'mocha';
+
+describe('Verify stryker has ran correctly', () => {
+
+  function expectExists(fileName: string) {
+    expect(fs.existsSync(fileName), `Missing ${fileName}!`).true;
+  }
+
+  it('should report html files', () => {
+    expectExists('reports/mutation/html/index.html');
+    expectExists('reports/mutation/html/mutation-test-elements.js');
+    expectExists('reports/mutation/html/stryker-80x80.png');
+    expectExists('reports/mutation/html/bind-mutation-test-report.js');
+  });
+
+  it('should have a clear text report', () => {
+    expectExists('reports/stdout.txt');
+  });
+
+  describe('clearText report', () => {
+
+    let stdout: string;
+    beforeEach(async () => {
+      stdout = await fs.promises.readFile('reports/stdout.txt', 'utf8');
+    })
+
+    it('should report NoCoverage mutants', () => {
+      expect(stdout).matches(createNoCoverageMutantRegex());
+    });
+
+    it('should report Survived mutants', () => {
+      expect(stdout).matches(createSurvivedMutantRegex());
+    });
+
+    it('should report average tests per mutant', () => {
+      expect(stdout).contains('Ran 0.80 tests per mutant on average.');
+    });
+
+    it('should report the clearText table', () => {
+      const clearTextTableRegex = createClearTextTableSummaryRowRegex();
+      expect(stdout).matches(clearTextTableRegex);
+    });
+
+    it('should finish up with the clear text report', () => {
+      const clearTextTableRegex = createClearTextTableSummaryRowRegex();
+      const survivedMutantRegex = createSurvivedMutantRegex();
+      const indexOfSurvivedMutant = survivedMutantRegex.exec(stdout).index;
+      const indexOfClearTextTable = clearTextTableRegex.exec(stdout).index;
+      expect(indexOfSurvivedMutant).lessThan(indexOfClearTextTable);
+    });
+  })
+
+});
+
+const createNoCoverageMutantRegex = () => /#6\.\s*\[NoCoverage\]/;
+
+const createSurvivedMutantRegex = () => /#20\.\s*\[Survived\]/;
+
+const createClearTextTableSummaryRowRegex = () => /All files\s*\|\s*64\.00\s*\|\s*16\s*\|\s*0\s*\|\s*1\s*\|\s*8\s*\|\s*0\s*\|/;

packages/core/src/reporters/MutationTestReportHelper.ts

@@ -59,18 +59,23 @@ export class MutationTestReportHelper {
       case MutantRunStatus.Error:
         return this.reportOne<InvalidMutantResult>(mutant, { status: MutantStatus.RuntimeError, errorMessage: result.errorMessage });
       case MutantRunStatus.Killed:
-        return this.reportOne<KilledMutantResult>(mutant, { status: MutantStatus.Killed, killedBy: this.testNamesById.get(result.killedBy)! });
+        return this.reportOne<KilledMutantResult>(mutant, {
+          status: MutantStatus.Killed,
+          nrOfTestsRan: result.nrOfTests,
+          killedBy: this.testNamesById.get(result.killedBy)!,
+        });
       case MutantRunStatus.Timeout:
         return this.reportOne<TimeoutMutantResult>(mutant, { status: MutantStatus.TimedOut });
       case MutantRunStatus.Survived:
         return this.reportOne<UndetectedMutantResult>(mutant, {
           status: MutantStatus.Survived,
+          nrOfTestsRan: result.nrOfTests,
           testFilter: testFilter ? this.dryRunResult.tests.filter((t) => testFilter.includes(t.id)).map((t) => t.name) : undefined,
         });
     }
   }
 
-  private reportOne<T extends MutantResult>(mutant: Mutant, additionalFields: Omit<T, keyof BaseMutantResult>) {
+  private reportOne<T extends MutantResult>(mutant: Mutant, additionalFields: Omit<T, keyof BaseMutantResult> & { nrOfTestsRan?: number }) {
     const originalFileTextContent = this.inputFiles.filesToMutate.find((fileToMutate) => fileToMutate.name === mutant.fileName)!.textContent;
 
     const mutantResult = {

packages/core/test/unit/reporters/MutationTestReportHelper.spec.ts

@@ -306,13 +306,14 @@ describe(MutationTestReportHelper.name, () => {
       // Act
       const actual = sut.reportMutantRunResult(
         createMutantTestCoverage({ mutant: factory.mutant({ fileName: 'add.js' }) }),
-        factory.killedMutantRunResult({ killedBy: '1' })
+        factory.killedMutantRunResult({ killedBy: '1', nrOfTests: 42 })
       );
 
       // Assert
       const expected: Partial<KilledMutantResult> = {
         status: MutantStatus.Killed,
         killedBy: 'foo should be bar',
+        nrOfTestsRan: 42,
       };
       expect(actual).deep.include(expected);
     });
@@ -360,13 +361,14 @@ describe(MutationTestReportHelper.name, () => {
       // Act
       const actual = sut.reportMutantRunResult(
         createMutantTestCoverage({ mutant: factory.mutant({ fileName: 'add.js' }), testFilter: ['1'] }),
-        factory.survivedMutantRunResult()
+        factory.survivedMutantRunResult({ nrOfTests: 4 })
       );
 
       // Assert
       const expected: Partial<UndetectedMutantResult> = {
         status: MutantStatus.Survived,
         testFilter: ['foo should be bar'],
+        nrOfTestsRan: 4,
       };
       expect(actual).deep.include(expected);
     });