feat(karma-runner): implement v2 test runner api

packages/karma-runner/.mocharc.jsonc

@@ -0,0 +1,3 @@
+{
+  "require": "test/helpers/setupTests.js"
+}

packages/karma-runner/.vscode/launch.json

@@ -1,53 +1,45 @@
 {
   "version": "0.2.0",
   "configurations": [
-    {
-      "type": "node",
-      "request": "launch",
-      "name": "Karma with angular",
-      "program": "${workspaceFolder}/run-angular.js",
-      "cwd": "C:\\z\\github\\nicojs\\angular-stryker-example",
-      "outFiles": [
-        "${workspaceRoot}/src/**/*.js",
-        "${workspaceRoot}/run-angular.js"
-      ]
-    },
     {
       "type": "node",
       "request": "launch",
       "name": "Unit tests",
       "program": "${workspaceRoot}/../../node_modules/mocha/bin/_mocha",
       "args": [
-        "--timeout",
-        "999999",
-        "--colors",
-        "${workspaceRoot}/test/helpers/**/*.js",
+        "--no-timeout",
         "${workspaceRoot}/test/unit/**/*.js"
       ],
+      "cwd": "${workspaceFolder}",
       "internalConsoleOptions": "openOnSessionStart",
       "outFiles": [
         "${workspaceRoot}/test/**/*.js",
         "${workspaceRoot}/src/**/*.js"
-      ]
+      ],
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
     },
     {
       "type": "node",
       "request": "launch",
       "name": "Integration tests",
       "program": "${workspaceRoot}/../../node_modules/mocha/bin/_mocha",
+      "cwd": "${workspaceFolder}",
       "args": [
-        "--timeout",
-        "999999",
-        "--colors",
-        "${workspaceRoot}/test/helpers/**/*.js",
+        "--no-timeout",
+        "--exit",
         "${workspaceRoot}/test/integration/**/*.js"
       ],
       "internalConsoleOptions": "openOnSessionStart",
       "outFiles": [
         "${workspaceRoot}/test/**/*.js",
         "${workspaceRoot}/src/**/*.js"
-      ]
+      ],      
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
     }
   ],
   "compounds": []
-}
+}

packages/karma-runner/package.json

@@ -6,8 +6,8 @@
   "scripts": {
     "test": "nyc --exclude-after-remap=false --check-coverage --reporter=html --report-dir=reports/coverage --lines 80 --functions 80 --branches 75 npm run mocha",
     "mocha": "npm run test:unit && npm run test:integration",
-    "test:unit": "mocha \"test/helpers/**/*.js\" \"test/unit/**/*.js\"",
-    "test:integration": "mocha --timeout 30000 --exit \"test/helpers/**/*.js\" \"test/integration/**/*.js\"",
+    "test:unit": "mocha \"test/unit/**/*.js\"",
+    "test:integration": "mocha --timeout 30000 --exit \"test/integration/**/*.js\"",
     "stryker": "node ../core/bin/stryker run"
   },
   "repository": {
@@ -32,8 +32,6 @@
   },
   "homepage": "https://github.com/stryker-mutator/stryker/tree/master/packages/karma-runner#readme",
   "devDependencies": {
-    "@stryker-mutator/jasmine-framework": "^3.3.0",
-    "@stryker-mutator/mocha-framework": "^3.3.0",
     "@stryker-mutator/test-helpers": "^3.3.0",
     "@stryker-mutator/util": "^3.3.0",
     "@types/express": "~4.17.0",

packages/karma-runner/src/KarmaTestRunner.ts

@@ -1,27 +1,35 @@
 import { StrykerOptions } from '@stryker-mutator/api/core';
 import { Logger, LoggerFactoryMethod } from '@stryker-mutator/api/logging';
 import { commonTokens, tokens } from '@stryker-mutator/api/plugin';
-import { CoverageCollection, CoveragePerTestResult, RunResult, RunStatus, TestResult, TestRunner } from '@stryker-mutator/api/test_runner';
 import * as karma from 'karma';
 
 import strykerKarmaConf = require('./starters/stryker-karma.conf');
 import { StrykerKarmaSetup } from '../src-generated/karma-runner-options';
+import {
+  TestRunner2,
+  TestResult,
+  MutantCoverage,
+  RunStatus,
+  DryRunOptions,
+  MutantRunOptions,
+  DryRunResult,
+  MutantRunResult,
+  toMutantRunResult,
+} from '../../api/test_runner2';
 
 import ProjectStarter from './starters/ProjectStarter';
-import StrykerReporter from './StrykerReporter';
-import TestHooksMiddleware from './TestHooksMiddleware';
+import StrykerReporter from './karma-plugins/StrykerReporter';
 import { KarmaRunnerOptionsWithStrykerOptions } from './KarmaRunnerOptionsWithStrykerOptions';
+import TestHooksMiddleware from './karma-plugins/TestHooksMiddleware';
 
 export interface ConfigOptions extends karma.ConfigOptions {
   detached?: boolean;
 }
 
-export default class KarmaTestRunner implements TestRunner {
+export default class KarmaTestRunner implements TestRunner2 {
   private currentTestResults: TestResult[];
-  private currentErrorMessages: string[];
-  private currentCoverageReport?: CoverageCollection | CoveragePerTestResult;
-  private currentRunStatus: RunStatus;
-  private readonly testHooksMiddleware = TestHooksMiddleware.instance;
+  private currentErrorMessage: string | undefined;
+  private currentCoverageReport?: MutantCoverage;
   private readonly starter: ProjectStarter;
   public port: undefined | number;
 
@@ -32,27 +40,31 @@ export default class KarmaTestRunner implements TestRunner {
     this.setGlobals(setup, getLogger);
     this.cleanRun();
     this.listenToServerStart();
-    this.listenToRunComplete();
     this.listenToSpecComplete();
     this.listenToCoverage();
     this.listenToError();
   }
 
-  public init(): Promise<void> {
+  public async init(): Promise<void> {
     return new Promise((res, rej) => {
-      StrykerReporter.instance.once('browsers_ready', res);
-      this.starter
-        .start()
-        .then(() => {
-          /*noop*/
-        })
-        .catch(rej);
+      StrykerReporter.instance.once('browsers_ready', () => res());
+      this.starter.start().catch(rej);
     });
   }
 
-  public async run({ testHooks }: { testHooks?: string }): Promise<RunResult> {
-    this.testHooksMiddleware.currentTestHooks = testHooks || '';
-    if (this.currentRunStatus !== RunStatus.Error) {
+  public dryRun(options: DryRunOptions): Promise<DryRunResult> {
+    TestHooksMiddleware.instance.configureCoverageAnalysis(options.coverageAnalysis);
+    return this.run();
+  }
+
+  public async mutantRun(options: MutantRunOptions): Promise<MutantRunResult> {
+    TestHooksMiddleware.instance.configureActiveMutant(options);
+    const dryRunResult = await this.run();
+    return toMutantRunResult(dryRunResult);
+  }
+
+  private async run(): Promise<DryRunResult> {
+    if (!this.currentErrorMessage) {
       // Only run when there was no compile error
       // An compile error can happen in case of angular-cli
       await this.runServer();
@@ -79,9 +91,8 @@ export default class KarmaTestRunner implements TestRunner {
 
   private cleanRun() {
     this.currentTestResults = [];
-    this.currentErrorMessages = [];
+    this.currentErrorMessage = undefined;
     this.currentCoverageReport = undefined;
-    this.currentRunStatus = RunStatus.Complete;
   }
 
   // Don't use dispose() to stop karma (using karma.stopper.stop)
@@ -100,24 +111,17 @@ export default class KarmaTestRunner implements TestRunner {
   }
 
   private listenToCoverage() {
-    StrykerReporter.instance.on('coverage_report', (coverageReport: CoverageCollection | CoveragePerTestResult) => {
+    StrykerReporter.instance.on('coverage_report', (coverageReport: MutantCoverage) => {
       this.currentCoverageReport = coverageReport;
     });
   }
 
-  private listenToRunComplete() {
-    StrykerReporter.instance.on('run_complete', (runStatus: RunStatus) => {
-      this.currentRunStatus = runStatus;
-    });
-  }
-
   private listenToError() {
     StrykerReporter.instance.on('browser_error', (error: string) => {
-      this.currentErrorMessages.push(error);
+      this.currentErrorMessage = error;
     });
     StrykerReporter.instance.on('compile_error', (errors: string[]) => {
-      errors.forEach((error) => this.currentErrorMessages.push(error));
-      this.currentRunStatus = RunStatus.Error;
+      this.currentErrorMessage = errors.join(', ');
     });
   }
 
@@ -130,25 +134,18 @@ export default class KarmaTestRunner implements TestRunner {
     });
   }
 
-  private collectRunResult(): RunResult {
-    return {
-      coverage: this.currentCoverageReport,
-      errorMessages: this.currentErrorMessages,
-      status: this.determineRunState(),
-      tests: this.currentTestResults,
-    };
-  }
-
-  private determineRunState() {
-    // Karma will report an Error if no tests had executed.
-    // This is not an "error" in Stryker terms
-    if (this.currentRunStatus === RunStatus.Error && !this.currentErrorMessages.length && !this.currentTestResults.length) {
-      return RunStatus.Complete;
-    } else if (this.currentErrorMessages.length) {
-      // Karma will return Complete when there are runtime errors
-      return RunStatus.Error;
+  private collectRunResult(): DryRunResult {
+    if (this.currentErrorMessage) {
+      return {
+        status: RunStatus.Error,
+        errorMessage: this.currentErrorMessage,
+      };
     } else {
-      return this.currentRunStatus;
+      return {
+        status: RunStatus.Complete,
+        tests: this.currentTestResults,
+        mutantCoverage: this.currentCoverageReport,
+      };
     }
   }
 }

packages/karma-runner/src/index.ts

@@ -4,5 +4,5 @@ import * as strykerValidationSchema from '../schema/karma-runner-options.json';
 
 import KarmaTestRunner from './KarmaTestRunner';
 
-export const strykerPlugins = [declareClassPlugin(PluginKind.TestRunner, 'karma', KarmaTestRunner)];
+export const strykerPlugins = [declareClassPlugin(PluginKind.TestRunner2, 'karma', KarmaTestRunner)];
 export { strykerValidationSchema };

packages/karma-runner/src/karma-plugins/StrykerMutantCoverageAdapter.ts

@@ -0,0 +1,16 @@
+// THIS FILE IS LOADED IN THE BROWSER AND SHOULD NOT BE A NODE / ES MODULE
+
+// @ts-expect-error
+const originalComplete = window.__karma__.complete.bind(window.__karma__);
+// @ts-expect-error
+window.__karma__.complete = (...args) => {
+  // @ts-expect-error
+  if (window.__strykerShouldReportCoverage__) {
+    if (!args.length) {
+      args.push({});
+    }
+    // @ts-expect-error
+    args[0].mutantCoverage = window.__mutantCoverage__;
+  }
+  originalComplete(...args);
+};

packages/karma-runner/src/StrykerReporter.ts → packages/karma-runner/src/karma-plugins/StrykerReporter.ts

@@ -1,6 +1,6 @@
 import { EventEmitter } from 'events';
 
-import { CoverageCollection, CoverageCollectionPerTest, RunStatus, TestResult, TestStatus } from '@stryker-mutator/api/test_runner';
+import { RunStatus, TestResult, TestStatus, MutantCoverage } from '@stryker-mutator/api/test_runner2';
 import * as karma from 'karma';
 
 export interface KarmaSpec {
@@ -42,18 +42,31 @@ export default class StrykerReporter extends EventEmitter implements karma.Repor
 
   public readonly onSpecComplete = (_browser: any, spec: KarmaSpec) => {
     const name = spec.suite.reduce((name, suite) => name + suite + ' ', '') + spec.description;
-    let status = TestStatus.Failed;
+    const id = spec.id || name;
+    let testResult: TestResult;
     if (spec.skipped) {
-      status = TestStatus.Skipped;
+      testResult = {
+        id,
+        name,
+        timeSpentMs: spec.time,
+        status: TestStatus.Skipped,
+      };
     } else if (spec.success) {
-      status = TestStatus.Success;
+      testResult = {
+        id,
+        name,
+        timeSpentMs: spec.time,
+        status: TestStatus.Success,
+      };
+    } else {
+      testResult = {
+        id,
+        name,
+        timeSpentMs: spec.time,
+        status: TestStatus.Failed,
+        failureMessage: spec.log.join(','),
+      };
     }
-    const testResult: TestResult = {
-      failureMessages: spec.log,
-      name,
-      status,
-      timeSpentMs: spec.time,
-    };
     this.emit('test_result', testResult);
   };
 
@@ -65,8 +78,8 @@ export default class StrykerReporter extends EventEmitter implements karma.Repor
     this.emit('load_error', ...args);
   };
 
-  public readonly onBrowserComplete = (_browser: any, result: { coverage: CoverageCollection | CoverageCollectionPerTest }) => {
-    this.emit('coverage_report', result.coverage);
+  public readonly onBrowserComplete = (_browser: any, result: { mutantCoverage: MutantCoverage }) => {
+    this.emit('coverage_report', result.mutantCoverage);
   };
 
   public readonly onBrowsersReady = () => {