From 582e01befe7ce2effdcde86f2c3123ccaff89c18 Mon Sep 17 00:00:00 2001 From: Nico Jansen Date: Fri, 4 Sep 2020 11:14:03 +0200 Subject: [PATCH] fix(reporters): correctly report avg tests/mutants (#2458) Correctly report the average tests/mutant metric in the clear-text reporter --- e2e/test/mocha-javascript/verify/verify.ts | 6 -- e2e/test/reporters-e2e/.mocharc.jsonc | 4 ++ e2e/test/reporters-e2e/package-lock.json | 5 ++ e2e/test/reporters-e2e/package.json | 15 +++++ e2e/test/reporters-e2e/src/Add.js | 26 ++++++++ e2e/test/reporters-e2e/src/Circle.js | 8 +++ e2e/test/reporters-e2e/stryker.conf.json | 11 ++++ .../reporters-e2e/test/helpers/testSetup.js | 5 ++ e2e/test/reporters-e2e/test/unit/AddSpec.js | 52 ++++++++++++++++ .../reporters-e2e/test/unit/CircleSpec.js | 13 ++++ e2e/test/reporters-e2e/verify/verify.ts | 61 +++++++++++++++++++ .../src/reporters/MutationTestReportHelper.ts | 9 ++- .../MutationTestReportHelper.spec.ts | 6 +- 13 files changed, 211 insertions(+), 10 deletions(-) create mode 100644 e2e/test/reporters-e2e/.mocharc.jsonc create mode 100644 e2e/test/reporters-e2e/package-lock.json create mode 100644 e2e/test/reporters-e2e/package.json create mode 100644 e2e/test/reporters-e2e/src/Add.js create mode 100644 e2e/test/reporters-e2e/src/Circle.js create mode 100644 e2e/test/reporters-e2e/stryker.conf.json create mode 100644 e2e/test/reporters-e2e/test/helpers/testSetup.js create mode 100644 e2e/test/reporters-e2e/test/unit/AddSpec.js create mode 100644 e2e/test/reporters-e2e/test/unit/CircleSpec.js create mode 100644 e2e/test/reporters-e2e/verify/verify.ts diff --git a/e2e/test/mocha-javascript/verify/verify.ts b/e2e/test/mocha-javascript/verify/verify.ts index 8c0d40228b..2ef7726bc7 100644 --- a/e2e/test/mocha-javascript/verify/verify.ts +++ b/e2e/test/mocha-javascript/verify/verify.ts @@ -25,10 +25,4 @@ describe('Verify stryker has ran correctly', () => { }); }); - it('should report html files', () => { - expectExists('reports/mutation/html/index.html'); - expectExists('reports/mutation/html/mutation-test-elements.js'); - expectExists('reports/mutation/html/stryker-80x80.png'); - expectExists('reports/mutation/html/bind-mutation-test-report.js'); - }); }); diff --git a/e2e/test/reporters-e2e/.mocharc.jsonc b/e2e/test/reporters-e2e/.mocharc.jsonc new file mode 100644 index 0000000000..3c2e5a6a9d --- /dev/null +++ b/e2e/test/reporters-e2e/.mocharc.jsonc @@ -0,0 +1,4 @@ +{ + "require": "./test/helpers/testSetup.js", + "spec": ["test/unit/*.js"] +} \ No newline at end of file diff --git a/e2e/test/reporters-e2e/package-lock.json b/e2e/test/reporters-e2e/package-lock.json new file mode 100644 index 0000000000..3900182e5c --- /dev/null +++ b/e2e/test/reporters-e2e/package-lock.json @@ -0,0 +1,5 @@ +{ + "name": "mocha-mocha", + "version": "0.0.0", + "lockfileVersion": 1 +} diff --git a/e2e/test/reporters-e2e/package.json b/e2e/test/reporters-e2e/package.json new file mode 100644 index 0000000000..a1576477fc --- /dev/null +++ b/e2e/test/reporters-e2e/package.json @@ -0,0 +1,15 @@ +{ + "name": "reporters-e2e", + "version": "0.0.0", + "private": true, + "description": "A module to perform an integration test", + "main": "index.js", + "scripts": { + "pretest": "rimraf \"reports\"", + "test": "mkdir -p reports && stryker run > reports/stdout.txt", + "test:unit": "mocha", + "posttest": "mocha --no-config --require ../../tasks/ts-node-register.js verify/*.ts" + }, + "author": "", + "license": "ISC" +} diff --git a/e2e/test/reporters-e2e/src/Add.js b/e2e/test/reporters-e2e/src/Add.js new file mode 100644 index 0000000000..0d6e64385c --- /dev/null +++ b/e2e/test/reporters-e2e/src/Add.js @@ -0,0 +1,26 @@ +module.exports.add = function(num1, num2) { + return num1 + num2; +}; + +module.exports.addOne = function(number) { + number++; + return number; +}; + +module.exports.negate = function(number) { + return -number; +}; + +module.exports.notCovered = function(number) { + return number > 10; +}; + +module.exports.isNegativeNumber = function(number) { + var isNegative = false; + if(number < 0){ + isNegative = true; + } + return isNegative; +}; + + diff --git a/e2e/test/reporters-e2e/src/Circle.js b/e2e/test/reporters-e2e/src/Circle.js new file mode 100644 index 0000000000..c1660c2d3f --- /dev/null +++ b/e2e/test/reporters-e2e/src/Circle.js @@ -0,0 +1,8 @@ +module.exports.getCircumference = function(radius) { + //Function to test multiple math mutations in a single function. + return 2 * Math.PI * radius; +}; + +module.exports.untestedFunction = function() { + var i = 5 / 2 * 3; +}; diff --git a/e2e/test/reporters-e2e/stryker.conf.json b/e2e/test/reporters-e2e/stryker.conf.json new file mode 100644 index 0000000000..fbb9dff9c5 --- /dev/null +++ b/e2e/test/reporters-e2e/stryker.conf.json @@ -0,0 +1,11 @@ +{ + "$schema": "../../node_modules/@stryker-mutator/core/schema/stryker-schema.json", + "testRunner": "mocha", + "concurrency": 2, + "coverageAnalysis": "perTest", + "reporters": ["clear-text", "html", "event-recorder"], + "plugins": [ + "@stryker-mutator/mocha-runner" + ], + "allowConsoleColors": false +} diff --git a/e2e/test/reporters-e2e/test/helpers/testSetup.js b/e2e/test/reporters-e2e/test/helpers/testSetup.js new file mode 100644 index 0000000000..993b68e6fc --- /dev/null +++ b/e2e/test/reporters-e2e/test/helpers/testSetup.js @@ -0,0 +1,5 @@ +exports.mochaHooks = { + beforeAll() { + global.expect = require('chai').expect; + } +} diff --git a/e2e/test/reporters-e2e/test/unit/AddSpec.js b/e2e/test/reporters-e2e/test/unit/AddSpec.js new file mode 100644 index 0000000000..d37ba16e88 --- /dev/null +++ b/e2e/test/reporters-e2e/test/unit/AddSpec.js @@ -0,0 +1,52 @@ +var addModule = require('../../src/Add'); +var add = addModule.add; +var addOne = addModule.addOne; +var isNegativeNumber = addModule.isNegativeNumber; +var negate = addModule.negate; +var notCovered = addModule.notCovered; + +describe('Add', function() { + it('should be able to add two numbers', function() { + var num1 = 2; + var num2 = 5; + var expected = num1 + num2; + + var actual = add(num1, num2); + + expect(actual).to.be.equal(expected); + }); + + it('should be able 1 to a number', function() { + var number = 2; + var expected = 3; + + var actual = addOne(number); + + expect(actual).to.be.equal(expected); + }); + + it('should be able negate a number', function() { + var number = 2; + var expected = -2; + + var actual = negate(number); + + expect(actual).to.be.equal(expected); + }); + + it('should be able to recognize a negative number', function() { + var number = -2; + + var isNegative = isNegativeNumber(number); + + expect(isNegative).to.be.true; + }); + + it('should be able to recognize that 0 is not a negative number', function() { + var number = 0; + + var isNegative = isNegativeNumber(number); + + expect(isNegative).to.be.false; + }); +}); diff --git a/e2e/test/reporters-e2e/test/unit/CircleSpec.js b/e2e/test/reporters-e2e/test/unit/CircleSpec.js new file mode 100644 index 0000000000..065c82220b --- /dev/null +++ b/e2e/test/reporters-e2e/test/unit/CircleSpec.js @@ -0,0 +1,13 @@ +var circleModule = require('../../src/Circle'); +var getCircumference = circleModule.getCircumference; + +describe('Circle', function() { + it('should have a circumference of 2PI when the radius is 1', function() { + var radius = 1; + var expectedCircumference = 2 * Math.PI; + + var circumference = getCircumference(radius); + + expect(circumference).to.be.equal(expectedCircumference); + }); +}); diff --git a/e2e/test/reporters-e2e/verify/verify.ts b/e2e/test/reporters-e2e/verify/verify.ts new file mode 100644 index 0000000000..50be0b4b5b --- /dev/null +++ b/e2e/test/reporters-e2e/verify/verify.ts @@ -0,0 +1,61 @@ +import { expect } from 'chai'; +import * as fs from 'fs'; +import { describe } from 'mocha'; + +describe('Verify stryker has ran correctly', () => { + + function expectExists(fileName: string) { + expect(fs.existsSync(fileName), `Missing ${fileName}!`).true; + } + + it('should report html files', () => { + expectExists('reports/mutation/html/index.html'); + expectExists('reports/mutation/html/mutation-test-elements.js'); + expectExists('reports/mutation/html/stryker-80x80.png'); + expectExists('reports/mutation/html/bind-mutation-test-report.js'); + }); + + it('should have a clear text report', () => { + expectExists('reports/stdout.txt'); + }); + + describe('clearText report', () => { + + let stdout: string; + beforeEach(async () => { + stdout = await fs.promises.readFile('reports/stdout.txt', 'utf8'); + }) + + it('should report NoCoverage mutants', () => { + expect(stdout).matches(createNoCoverageMutantRegex()); + }); + + it('should report Survived mutants', () => { + expect(stdout).matches(createSurvivedMutantRegex()); + }); + + it('should report average tests per mutant', () => { + expect(stdout).contains('Ran 0.80 tests per mutant on average.'); + }); + + it('should report the clearText table', () => { + const clearTextTableRegex = createClearTextTableSummaryRowRegex(); + expect(stdout).matches(clearTextTableRegex); + }); + + it('should finish up with the clear text report', () => { + const clearTextTableRegex = createClearTextTableSummaryRowRegex(); + const survivedMutantRegex = createSurvivedMutantRegex(); + const indexOfSurvivedMutant = survivedMutantRegex.exec(stdout).index; + const indexOfClearTextTable = clearTextTableRegex.exec(stdout).index; + expect(indexOfSurvivedMutant).lessThan(indexOfClearTextTable); + }); + }) + +}); + +const createNoCoverageMutantRegex = () => /#6\.\s*\[NoCoverage\]/; + +const createSurvivedMutantRegex = () => /#20\.\s*\[Survived\]/; + +const createClearTextTableSummaryRowRegex = () => /All files\s*\|\s*64\.00\s*\|\s*16\s*\|\s*0\s*\|\s*1\s*\|\s*8\s*\|\s*0\s*\|/; diff --git a/packages/core/src/reporters/MutationTestReportHelper.ts b/packages/core/src/reporters/MutationTestReportHelper.ts index c082738250..b851847345 100644 --- a/packages/core/src/reporters/MutationTestReportHelper.ts +++ b/packages/core/src/reporters/MutationTestReportHelper.ts @@ -59,18 +59,23 @@ export class MutationTestReportHelper { case MutantRunStatus.Error: return this.reportOne(mutant, { status: MutantStatus.RuntimeError, errorMessage: result.errorMessage }); case MutantRunStatus.Killed: - return this.reportOne(mutant, { status: MutantStatus.Killed, killedBy: this.testNamesById.get(result.killedBy)! }); + return this.reportOne(mutant, { + status: MutantStatus.Killed, + nrOfTestsRan: result.nrOfTests, + killedBy: this.testNamesById.get(result.killedBy)!, + }); case MutantRunStatus.Timeout: return this.reportOne(mutant, { status: MutantStatus.TimedOut }); case MutantRunStatus.Survived: return this.reportOne(mutant, { status: MutantStatus.Survived, + nrOfTestsRan: result.nrOfTests, testFilter: testFilter ? this.dryRunResult.tests.filter((t) => testFilter.includes(t.id)).map((t) => t.name) : undefined, }); } } - private reportOne(mutant: Mutant, additionalFields: Omit) { + private reportOne(mutant: Mutant, additionalFields: Omit & { nrOfTestsRan?: number }) { const originalFileTextContent = this.inputFiles.filesToMutate.find((fileToMutate) => fileToMutate.name === mutant.fileName)!.textContent; const mutantResult = { diff --git a/packages/core/test/unit/reporters/MutationTestReportHelper.spec.ts b/packages/core/test/unit/reporters/MutationTestReportHelper.spec.ts index 2d94a5e368..461b3038a5 100644 --- a/packages/core/test/unit/reporters/MutationTestReportHelper.spec.ts +++ b/packages/core/test/unit/reporters/MutationTestReportHelper.spec.ts @@ -306,13 +306,14 @@ describe(MutationTestReportHelper.name, () => { // Act const actual = sut.reportMutantRunResult( createMutantTestCoverage({ mutant: factory.mutant({ fileName: 'add.js' }) }), - factory.killedMutantRunResult({ killedBy: '1' }) + factory.killedMutantRunResult({ killedBy: '1', nrOfTests: 42 }) ); // Assert const expected: Partial = { status: MutantStatus.Killed, killedBy: 'foo should be bar', + nrOfTestsRan: 42, }; expect(actual).deep.include(expected); }); @@ -360,13 +361,14 @@ describe(MutationTestReportHelper.name, () => { // Act const actual = sut.reportMutantRunResult( createMutantTestCoverage({ mutant: factory.mutant({ fileName: 'add.js' }), testFilter: ['1'] }), - factory.survivedMutantRunResult() + factory.survivedMutantRunResult({ nrOfTests: 4 }) ); // Assert const expected: Partial = { status: MutantStatus.Survived, testFilter: ['foo should be bar'], + nrOfTestsRan: 4, }; expect(actual).deep.include(expected); });