feat(api): add new test runner api

packages/api/.vscode/launch.json

@@ -1,24 +1,6 @@
 {
   "version": "0.2.0",
   "configurations": [
-    {
-      "type": "node",
-      "request": "launch",
-      "name": "Integration tests",
-      "program": "${workspaceFolder}/../../node_modules/mocha/bin/_mocha",
-      "args": [
-        "--timeout",
-        "999999",
-        "--colors",
-        "${workspaceFolder}/test/helpers/**/*.js",
-        "${workspaceFolder}/test/integration/**/*.js"
-      ],
-      "internalConsoleOptions": "openOnSessionStart",
-      "outFiles": [
-        "${workspaceRoot}/test/**/*.js",
-        "${workspaceRoot}/src/**/*.js"
-      ]
-    },
     {
       "type": "node",
       "request": "launch",
@@ -35,7 +17,10 @@
       "outFiles": [
         "${workspaceRoot}/test/**/*.js",
         "${workspaceRoot}/src/**/*.js"
+      ],
+      "skipFiles": [
+        "<node_internals>/**"
       ]
     }
   ]
-}
+}

packages/api/schema/stryker-core.json

@@ -26,6 +26,15 @@
         "Trace"
       ]
     },
+    "coverageAnalysis": {
+      "title": "CoverageAnalysis",
+      "type": "string",
+      "enum": [
+        "off",
+        "all",
+        "perTest"
+      ]
+    },
     "reportType": {
       "title": "ReportType",
       "type": "string",
@@ -201,13 +210,8 @@
       "default": true
     },
     "coverageAnalysis": {
+      "$ref": "#/definitions/coverageAnalysis",
       "description": "Indicates which coverage analysis strategy to use. During mutation testing, stryker will try to only run the tests that cover a particular line of code.\n\n'perTest': Analyse coverage per test.\n'all': Analyse the coverage for the entire test suite.\n'off' (default): Don't use coverage analysis",
-      "type": "string",
-      "enum": [
-        "off",
-        "all",
-        "perTest"
-      ],
       "default": "off"
     },
     "clearTextReporter": {

packages/api/src/plugin/Contexts.ts

@@ -47,6 +47,7 @@ export interface PluginContexts {
   [PluginKind.Reporter]: OptionsContext;
   [PluginKind.TestFramework]: OptionsContext;
   [PluginKind.TestRunner]: TestRunnerPluginContext;
+  [PluginKind.TestRunner2]: TestRunnerPluginContext;
   [PluginKind.Transpiler]: TranspilerPluginContext;
   [PluginKind.Checker]: OptionsContext;
 }

packages/api/src/plugin/PluginKind.ts

@@ -9,6 +9,7 @@ export enum PluginKind {
   OptionsEditor = 'OptionsEditor',
   Checker = 'Checker',
   TestRunner = 'TestRunner',
+  TestRunner2 = 'TestRunner2',
   TestFramework = 'TestFramework',
   Transpiler = 'Transpiler',
   Mutator = 'Mutator',

packages/api/src/plugin/Plugins.ts

@@ -7,6 +7,7 @@ import { TestFramework } from '../../test_framework';
 import { TestRunner } from '../../test_runner';
 import { Transpiler } from '../../transpile';
 import { OptionsEditor } from '../core/OptionsEditor';
+import { TestRunner2 } from '../../test_runner2';
 import { Checker } from '../../check';
 
 import { PluginContexts } from './Contexts';
@@ -90,6 +91,7 @@ export interface PluginInterfaces {
   [PluginKind.Reporter]: Reporter;
   [PluginKind.TestFramework]: TestFramework;
   [PluginKind.TestRunner]: TestRunner;
+  [PluginKind.TestRunner2]: TestRunner2;
   [PluginKind.Transpiler]: Transpiler;
   [PluginKind.Checker]: Checker;
 }

packages/api/src/test_runner2/DryRunResult.ts

@@ -0,0 +1,37 @@
+import { MutantCoverage } from './MutantCoverage';
+import { RunStatus } from './RunStatus';
+import { TestResult } from './TestResult';
+
+export type DryRunResult = CompleteDryRunResult | TimeoutDryRunResult | ErrorDryRunResult;
+
+export interface CompleteDryRunResult {
+  /**
+   * The individual test results.
+   */
+  tests: TestResult[];
+
+  mutantCoverage?: MutantCoverage;
+
+  /**
+   * The status of the run
+   */
+  status: RunStatus.Complete;
+}
+export interface TimeoutDryRunResult {
+  /**
+   * The status of the run
+   */
+  status: RunStatus.Timeout;
+}
+
+export interface ErrorDryRunResult {
+  /**
+   * The status of the run
+   */
+  status: RunStatus.Error;
+
+  /**
+   * If `state` is `error`, this collection should contain the error messages
+   */
+  errorMessage: string;
+}

packages/api/src/test_runner2/MutantCoverage.ts

@@ -0,0 +1,12 @@
+export interface MutantCoverage {
+  static: CoverageData;
+  perTest: CoveragePerTestId;
+}
+
+export interface CoveragePerTestId {
+  [testId: string]: CoverageData;
+}
+
+export interface CoverageData {
+  [mutantId: number]: number;
+}

packages/api/src/test_runner2/MutantRunResult.ts

@@ -0,0 +1,33 @@
+export enum MutantRunStatus {
+  Killed = 'killed',
+  Survived = 'survived',
+  Timeout = 'timeout',
+  Error = 'error',
+}
+
+export type MutantRunResult = KilledMutantRunResult | SurvivedMutantRunResult | TimeoutMutantRunResult | ErrorMutantRunResult;
+
+export interface TimeoutMutantRunResult {
+  status: MutantRunStatus.Timeout;
+}
+
+export interface KilledMutantRunResult {
+  status: MutantRunStatus.Killed;
+  /**
+   * The id of the test that killed this mutant
+   */
+  killedBy: string;
+  /**
+   * The failure message that was reported by the test
+   */
+  failureMessage: string;
+}
+
+export interface SurvivedMutantRunResult {
+  status: MutantRunStatus.Survived;
+}
+
+export interface ErrorMutantRunResult {
+  status: MutantRunStatus.Error;
+  errorMessage: string;
+}

packages/api/src/test_runner2/RunOptions.ts

@@ -0,0 +1,22 @@
+import { Mutant, CoverageAnalysis } from '../../core';
+
+export interface RunOptions {
+  /**
+   * The amount of time (in milliseconds) the TestRunner has to complete the test run before a timeout occurs.
+   */
+  timeout: number;
+}
+
+export interface DryRunOptions extends RunOptions {
+  /**
+   * Indicates whether or not mutant coverage should be collected.
+   */
+  coverageAnalysis: CoverageAnalysis;
+}
+
+export interface MutantRunOptions extends RunOptions {
+  testFilter?: string[];
+  activeMutant: Mutant;
+}
+
+export default RunOptions;

packages/api/src/test_runner2/RunStatus.ts

@@ -0,0 +1,14 @@
+export enum RunStatus {
+  /**
+   * Indicates that a test run is completed with failed or succeeded tests
+   */
+  Complete = 'complete',
+  /**
+   * Indicates that a test run cut off early with an error
+   */
+  Error = 'error',
+  /**
+   * Indicates that a test run timed out
+   */
+  Timeout = 'timeout',
+}

packages/api/src/test_runner2/TestResult.ts

@@ -0,0 +1,34 @@
+import { TestStatus } from './TestStatus';
+
+/**
+ * Indicates the result of a single test
+ */
+interface BaseTestResult {
+  /**
+   * The id of this test. Can be the name if the test runner doesn't have an 'id'
+   */
+  id: string;
+  /**
+   * The full human readable name of the test
+   */
+  name: string;
+  /**
+   * The time it took to run the test
+   */
+  timeSpentMs: number;
+}
+
+export interface FailedTestResult extends BaseTestResult {
+  status: TestStatus.Failed;
+  failureMessage: string;
+}
+
+export interface SkippedTestResult extends BaseTestResult {
+  status: TestStatus.Skipped;
+}
+
+export interface SuccessTestResult extends BaseTestResult {
+  status: TestStatus.Success;
+}
+
+export type TestResult = SuccessTestResult | FailedTestResult | SkippedTestResult;

packages/api/src/test_runner2/TestRunner.ts

@@ -0,0 +1,10 @@
+import { DryRunOptions, MutantRunOptions } from './RunOptions';
+import { DryRunResult } from './DryRunResult';
+import { MutantRunResult } from './MutantRunResult';
+
+export interface TestRunner2 {
+  init?(): Promise<void> | void;
+  dryRun(options: DryRunOptions): Promise<DryRunResult>;
+  mutantRun(options: MutantRunOptions): Promise<MutantRunResult>;
+  dispose?(): Promise<void> | void;
+}

packages/api/src/test_runner2/TestStatus.ts

@@ -0,0 +1,17 @@
+/**
+ * Indicates what the result of a single test was.
+ */
+export enum TestStatus {
+  /**
+   * The test succeeded
+   */
+  Success,
+  /**
+   * The test failed
+   */
+  Failed,
+  /**
+   * The test was skipped (not executed)
+   */
+  Skipped,
+}

packages/api/src/test_runner2/runResultHelpers.ts

@@ -0,0 +1,34 @@
+import { TestStatus } from './TestStatus';
+import { DryRunResult } from './DryRunResult';
+import { MutantRunResult } from './MutantRunResult';
+import { RunStatus } from './RunStatus';
+import { MutantRunStatus } from './MutantRunResult';
+import { FailedTestResult } from './TestResult';
+
+export function toMutantRunResult(dryRunResult: DryRunResult): MutantRunResult {
+  switch (dryRunResult.status) {
+    case RunStatus.Complete: {
+      const killedBy = dryRunResult.tests.find<FailedTestResult>((test): test is FailedTestResult => test.status === TestStatus.Failed);
+      if (killedBy) {
+        return {
+          status: MutantRunStatus.Killed,
+          failureMessage: killedBy.failureMessage,
+          killedBy: killedBy.id,
+        };
+      } else {
+        return {
+          status: MutantRunStatus.Survived,
+        };
+      }
+    }
+    case RunStatus.Error:
+      return {
+        status: MutantRunStatus.Error,
+        errorMessage: dryRunResult.errorMessage,
+      };
+    case RunStatus.Timeout:
+      return {
+        status: MutantRunStatus.Timeout,
+      };
+  }
+}

packages/api/test/unit/test_runner2/runResultHelpers.spec.ts

@@ -0,0 +1,56 @@
+import { expect } from 'chai';
+
+import { toMutantRunResult, RunStatus, MutantRunResult, MutantRunStatus } from '../../../test_runner2';
+import TestStatus from '../../../src/test_runner/TestStatus';
+
+describe('runResultHelpers', () => {
+  describe(toMutantRunResult.name, () => {
+    it('should convert "timeout" to "timeout"', () => {
+      const expected: MutantRunResult = { status: MutantRunStatus.Timeout };
+      expect(toMutantRunResult({ status: RunStatus.Timeout })).deep.eq(expected);
+    });
+
+    it('should convert "error" to "error"', () => {
+      const expected: MutantRunResult = { status: MutantRunStatus.Error, errorMessage: 'some error' };
+      expect(toMutantRunResult({ status: RunStatus.Error, errorMessage: 'some error' })).deep.eq(expected);
+    });
+
+    it('should report a failed test as "killed"', () => {
+      const expected: MutantRunResult = { status: MutantRunStatus.Killed, failureMessage: 'expected foo to be bar', killedBy: '42' };
+      expect(
+        toMutantRunResult({
+          status: RunStatus.Complete,
+          tests: [
+            { status: TestStatus.Success, id: 'success1', name: 'success1', timeSpentMs: 42 },
+            { status: TestStatus.Failed, id: '42', name: 'error', timeSpentMs: 42, failureMessage: 'expected foo to be bar' },
+            { status: TestStatus.Success, id: 'success2', name: 'success2', timeSpentMs: 42 },
+          ],
+        })
+      ).deep.eq(expected);
+    });
+
+    it('should report only succeeded tests as "survived"', () => {
+      const expected: MutantRunResult = { status: MutantRunStatus.Survived };
+      expect(
+        toMutantRunResult({
+          status: RunStatus.Complete,
+          tests: [
+            { status: TestStatus.Success, id: 'success1', name: 'success1', timeSpentMs: 42 },
+            { status: TestStatus.Success, id: '42', name: 'error', timeSpentMs: 42 },
+            { status: TestStatus.Success, id: 'success2', name: 'success2', timeSpentMs: 42 },
+          ],
+        })
+      ).deep.eq(expected);
+    });
+
+    it('should report an empty suite as "survived"', () => {
+      const expected: MutantRunResult = { status: MutantRunStatus.Survived };
+      expect(
+        toMutantRunResult({
+          status: RunStatus.Complete,
+          tests: [],
+        })
+      ).deep.eq(expected);
+    });
+  });
+});

packages/api/test_runner2.ts

@@ -0,0 +1,9 @@
+export * from './src/test_runner2/TestResult';
+export * from './src/test_runner2/TestRunner';
+export * from './src/test_runner2/TestStatus';
+export * from './src/test_runner2/DryRunResult';
+export * from './src/test_runner2/RunOptions';
+export * from './src/test_runner2/MutantCoverage';
+export * from './src/test_runner2/MutantRunResult';
+export * from './src/test_runner2/RunStatus';
+export * from './src/test_runner2/runResultHelpers';

packages/api/tsconfig.src.json

@@ -15,6 +15,7 @@
     "report.ts",
     "test_framework.ts",
     "test_runner.ts",
+    "test_runner2.ts",
     "transpile.ts",
     "plugin.ts"
   ],