feat(report): report test states (#2868)

Add test reporting to both the `clear-text` and `html` reporters (it was already supported by the JSON reporter). 

<table>
<tr>
<td>
<img src="https://user-images.githubusercontent.com/1828233/117065346-b1029b80-ad27-11eb-8d96-86c09bbd1194.png">
</td>
<td>
<img src="https://user-images.githubusercontent.com/1828233/117065413-c677c580-ad27-11eb-8998-cc416aa3ffea.png">
</td>
</table>

![image](https://user-images.githubusercontent.com/1828233/117066423-00959700-ad29-11eb-9d81-8beb3d0e5f2d.png)

There are a couple of things to keep in mind:

* Stryker will only be able to produce the [`Covering` test state](https://stryker-mutator.io/docs/mutation-testing-elements/mutant-states-and-metrics/#test-states-and-metrics) with `"coverageAnalysis": "perTest"`
  ![image](https://user-images.githubusercontent.com/1828233/117104495-85f66700-ad7c-11eb-9c50-250f503efd6b.png)
* Currently, only the `@stryker-mutator/jest-runner` supports marking the test files for each test. This means that all other test runners will simply produce a big list of tests:
  ![image](https://user-images.githubusercontent.com/1828233/117104637-ca820280-ad7c-11eb-97fb-ce9dcc700df0.png)

e2e/package.json

@@ -31,7 +31,7 @@
     "link-parent-bin": "^2.0.0",
     "load-grunt-tasks": "~5.1.0",
     "mocha": "~8.2.0",
-    "mutation-testing-metrics": "~1.6.2",
+    "mutation-testing-metrics": "1.7.2",
     "rxjs": "~6.5.3",
     "semver": "~6.3.0",
     "ts-jest": "~26.3.0",

e2e/test/karma-webpack-with-ts/package.json

@@ -8,8 +8,8 @@
   },
   "dependencies": {
     "lit-element": "~2.3.1",
-    "mutation-testing-metrics": "~1.6.2",
-    "mutation-testing-report-schema": "~1.6.0"
+    "mutation-testing-metrics": "1.7.2",
+    "mutation-testing-report-schema": "1.7.1"
   },
   "devDependencies": {
     "@types/webpack-env": "~1.15.2"

e2e/test/reporters-e2e/verify/verify.ts

@@ -27,6 +27,10 @@ describe('Verify stryker has ran correctly', () => {
       stdout = await fs.promises.readFile('reports/stdout.txt', 'utf8');
     })
 
+    it('should report all tests', () => {
+      expect(stdout).matches(createTestsRegex());
+    });
+
     it('should report NoCoverage mutants', () => {
       expect(stdout).matches(createNoCoverageMutantRegex());
     });
@@ -51,11 +55,11 @@ describe('Verify stryker has ran correctly', () => {
       const indexOfClearTextTable = clearTextTableRegex.exec(stdout).index;
       expect(indexOfSurvivedMutant).lessThan(indexOfClearTextTable);
     });
-  })
-
-
+  });
 });
 
+const createTestsRegex = () => /All tests\s*✓ Add should be able to add two numbers \(killed 2\)/;
+
 const createNoCoverageMutantRegex = () => /#6\.\s*\[NoCoverage\]/;
 
 const createSurvivedMutantRegex = () => /#20\.\s*\[Survived\]/;

packages/api/package.json

@@ -39,8 +39,8 @@
     "node": ">=10"
   },
   "dependencies": {
-    "mutation-testing-report-schema": "~1.6.0",
-    "mutation-testing-metrics": "~1.6.2",
+    "mutation-testing-report-schema": "1.7.1",
+    "mutation-testing-metrics": "1.7.2",
     "surrial": "~2.0.2",
     "tslib": "~2.2.0"
   },

packages/core/package.json

@@ -72,8 +72,8 @@
     "log4js": "~6.2.1",
     "minimatch": "~3.0.4",
     "mkdirp": "~1.0.3",
-    "mutation-testing-elements": "~1.6.2",
-    "mutation-testing-metrics": "~1.6.2",
+    "mutation-testing-elements": "1.7.2",
+    "mutation-testing-metrics": "1.7.2",
     "npm-run-path": "~4.0.1",
     "progress": "~2.0.0",
     "rimraf": "~3.0.0",

packages/core/src/reporters/clear-text-reporter.ts

@@ -1,11 +1,11 @@
 import os from 'os';
 
-import chalk from 'chalk';
+import chalk, { Color } from 'chalk';
 import { schema, Position, StrykerOptions } from '@stryker-mutator/api/core';
 import { Logger } from '@stryker-mutator/api/logging';
 import { commonTokens } from '@stryker-mutator/api/plugin';
 import { Reporter } from '@stryker-mutator/api/report';
-import { MetricsResult, MutantModel, TestModel, MutationTestMetricsResult } from 'mutation-testing-metrics';
+import { MetricsResult, MutantModel, TestModel, MutationTestMetricsResult, TestFileModel, TestMetrics, TestStatus } from 'mutation-testing-metrics';
 import { tokens } from 'typed-inject';
 
 import { plural } from '../utils/string-utils';
@@ -22,9 +22,13 @@ export class ClearTextReporter implements Reporter {
 
   private readonly out: NodeJS.WritableStream = process.stdout;
 
-  private writeLine(output?: string) {
+  private readonly writeLine = (output?: string) => {
     this.out.write(`${output ?? ''}${os.EOL}`);
-  }
+  };
+
+  private readonly writeDebugLine = (input: string) => {
+    this.log.debug(input);
+  };
 
   private configConsoleColor() {
     if (!this.options.allowConsoleColors) {
@@ -33,18 +37,53 @@ export class ClearTextReporter implements Reporter {
   }
 
   public onMutationTestReportReady(_report: schema.MutationTestResult, metrics: MutationTestMetricsResult): void {
+    this.writeLine();
+    this.reportAllTests(metrics);
     this.reportAllMutants(metrics);
     this.writeLine(new ClearTextScoreTable(metrics.systemUnderTestMetrics, this.options.thresholds).draw());
   }
 
+  private reportAllTests(metrics: MutationTestMetricsResult) {
+    function indent(depth: number) {
+      return new Array(depth).fill('  ').join('');
+    }
+    const formatTestLine = (test: TestModel, state: string): string => {
+      return `${this.color('grey', `${test.name}${test.location ? ` [line ${test.location.start.line}]` : ''}`)} (${state})`;
+    };
+
+    if (metrics.testMetrics) {
+      const reportTests = (currentResult: MetricsResult<TestFileModel, TestMetrics>, depth = 0) => {
+        const nameParts: string[] = [currentResult.name];
+        while (!currentResult.file && currentResult.childResults.length === 1) {
+          currentResult = currentResult.childResults[0];
+          nameParts.push(currentResult.name);
+        }
+        this.writeLine(`${indent(depth)}${nameParts.join('/')}`);
+        currentResult.file?.tests.forEach((test) => {
+          switch (test.status) {
+            case TestStatus.Killing:
+              this.writeLine(`${indent(depth + 1)}${this.color('greenBright', '✓')} ${formatTestLine(test, `killed ${test.killedMutants?.length}`)}`);
+              break;
+            case TestStatus.Covering:
+              this.writeLine(
+                `${indent(depth + 1)}${this.color('blueBright', '~')} ${formatTestLine(test, `covered ${test.coveredMutants?.length}`)}`
+              );
+              break;
+            case TestStatus.NotCovering:
+              this.writeLine(`${indent(depth + 1)}${this.color('redBright', '✘')} ${formatTestLine(test, 'covered 0')}`);
+              break;
+          }
+        });
+        currentResult.childResults.forEach((childResult) => reportTests(childResult, depth + 1));
+      };
+      reportTests(metrics.testMetrics);
+    }
+  }
+
   private reportAllMutants({ systemUnderTestMetrics }: MutationTestMetricsResult): void {
     this.writeLine();
     let totalTests = 0;
 
-    // use these functions in order to preserve the 'this` pointer
-    const logDebugFn = (input: string) => this.log.debug(input);
-    const writeLineFn = (input: string) => this.writeLine(input);
-
     const reportMutants = (metrics: MetricsResult[]) => {
       metrics.forEach((child) => {
         child.file?.mutants.forEach((result) => {
@@ -54,11 +93,11 @@ export class ClearTextReporter implements Reporter {
             case MutantStatus.Timeout:
             case MutantStatus.RuntimeError:
             case MutantStatus.CompileError:
-              this.reportMutantResult(result, logDebugFn);
+              this.reportMutantResult(result, this.writeDebugLine);
               break;
             case MutantStatus.Survived:
             case MutantStatus.NoCoverage:
-              this.reportMutantResult(result, writeLineFn);
+              this.reportMutantResult(result, this.writeLine);
               break;
             default:
           }
@@ -103,11 +142,14 @@ export class ClearTextReporter implements Reporter {
   }
 
   private colorSourceFileAndLocation(fileName: string, position: Position): string {
-    if (!this.options.clearTextReporter.allowColor) {
-      return `${fileName}:${position.line}:${position.column}`;
-    }
+    return [this.color('cyan', fileName), this.color('yellow', position.line), this.color('yellow', position.column)].join(':');
+  }
 
-    return [chalk.cyan(fileName), chalk.yellow(`${position.line}`), chalk.yellow(`${position.column}`)].join(':');
+  private color(color: typeof Color, ...text: unknown[]) {
+    if (this.options.clearTextReporter.allowColor) {
+      return chalk[color](...text);
+    }
+    return text.join('');
   }
 
   private logExecutedTests(tests: TestModel[], logImplementation: (input: string) => void) {