From a0b6055469f7c1b344d83aa006350cb65f6fcf35 Mon Sep 17 00:00:00 2001
From: Nico Jansen <jansennico@gmail.com>
Date: Tue, 9 Jun 2020 07:43:07 +0200
Subject: [PATCH 01/11] feat(api): add new test runner api

This adds a new test runner api to Stryker. Differences with the old API:

1. Explicit difference between a `dryRun` (initial test run) and a `mutantRun`
2. Code coverage interfaces are replaced with the simpler `MutantCoverage` interface.
3. The `MutantRunOptions` now has a `testFilter`, which replaces the old `TestFramework` api.
---
 .../api/src/test_runner2/MutantCoverage.ts    | 13 ++++++++
 packages/api/src/test_runner2/RunOptions.ts   | 23 +++++++++++++
 packages/api/src/test_runner2/RunResult.ts    | 32 +++++++++++++++++++
 packages/api/src/test_runner2/RunStatus.ts    | 14 ++++++++
 packages/api/src/test_runner2/TestResult.ts   | 23 +++++++++++++
 packages/api/src/test_runner2/TestRunner.ts   |  9 ++++++
 .../api/src/test_runner2/TestSelection.ts     |  4 +++
 packages/api/src/test_runner2/TestStatus.ts   | 19 +++++++++++
 packages/api/test_runner2.ts                  |  6 ++++
 packages/api/tsconfig.src.json                |  1 +
 tsconfig.lint.json                            | 14 ++++++--
 11 files changed, 156 insertions(+), 2 deletions(-)
 create mode 100644 packages/api/src/test_runner2/MutantCoverage.ts
 create mode 100644 packages/api/src/test_runner2/RunOptions.ts
 create mode 100644 packages/api/src/test_runner2/RunResult.ts
 create mode 100644 packages/api/src/test_runner2/RunStatus.ts
 create mode 100644 packages/api/src/test_runner2/TestResult.ts
 create mode 100644 packages/api/src/test_runner2/TestRunner.ts
 create mode 100644 packages/api/src/test_runner2/TestSelection.ts
 create mode 100644 packages/api/src/test_runner2/TestStatus.ts
 create mode 100644 packages/api/test_runner2.ts

diff --git a/packages/api/src/test_runner2/MutantCoverage.ts b/packages/api/src/test_runner2/MutantCoverage.ts
new file mode 100644
index 0000000000..8298a0856f
--- /dev/null
+++ b/packages/api/src/test_runner2/MutantCoverage.ts
@@ -0,0 +1,13 @@
+export type MutantCoverage = StaticCoverage & CoveragePerTestId;
+
+export interface StaticCoverage {
+  static: CoverageData;
+}
+
+export interface CoveragePerTestId {
+  [testId: number]: CoverageData;
+}
+
+export interface CoverageData {
+  [mutantId: number]: number;
+}
diff --git a/packages/api/src/test_runner2/RunOptions.ts b/packages/api/src/test_runner2/RunOptions.ts
new file mode 100644
index 0000000000..627725d410
--- /dev/null
+++ b/packages/api/src/test_runner2/RunOptions.ts
@@ -0,0 +1,23 @@
+import { TestSelection } from '../../test_framework';
+import { Mutant } from '../../mutant';
+
+export interface RunOptions {
+  /**
+   * The amount of time (in milliseconds) the TestRunner has to complete the test run before a timeout occurs.
+   */
+  timeout: number;
+}
+
+export interface DryRunOptions extends RunOptions {
+  /**
+   * Indicates whether or not mutant coverage should be collected.
+   */
+  reportMutantCoverage: boolean;
+}
+
+export interface MutantRunOptions extends RunOptions {
+  testFilter?: TestSelection[];
+  activeMutant: Mutant;
+}
+
+export default RunOptions;
diff --git a/packages/api/src/test_runner2/RunResult.ts b/packages/api/src/test_runner2/RunResult.ts
new file mode 100644
index 0000000000..fb4170f966
--- /dev/null
+++ b/packages/api/src/test_runner2/RunResult.ts
@@ -0,0 +1,32 @@
+import { MutantCoverage } from './MutantCoverage';
+import { RunStatus } from './RunStatus';
+import { TestResult } from './TestResult';
+
+/**
+ * Marker interface for a mutant run result.
+ */
+export interface MutantRunResult extends RunResult {}
+
+/**
+ * Represents the result of a test run.
+ */
+export interface RunResult {
+  /**
+   * The individual test results.
+   */
+  tests: TestResult[];
+
+  /**
+   * If `state` is `error`, this collection should contain the error messages
+   */
+  errorMessages?: string[];
+
+  /**
+   * The status of the run
+   */
+  status: RunStatus;
+}
+
+export interface DryRunResult extends RunResult {
+  mutationCoverage?: MutantCoverage;
+}
diff --git a/packages/api/src/test_runner2/RunStatus.ts b/packages/api/src/test_runner2/RunStatus.ts
new file mode 100644
index 0000000000..86fe77af14
--- /dev/null
+++ b/packages/api/src/test_runner2/RunStatus.ts
@@ -0,0 +1,14 @@
+export enum RunStatus {
+  /**
+   * Indicates that a test run is completed with failed or succeeded tests
+   */
+  Complete,
+  /**
+   * Indicates that a test run cut off early with an error
+   */
+  Error,
+  /**
+   * Indicates that a test run timed out
+   */
+  Timeout,
+}
diff --git a/packages/api/src/test_runner2/TestResult.ts b/packages/api/src/test_runner2/TestResult.ts
new file mode 100644
index 0000000000..35936863fa
--- /dev/null
+++ b/packages/api/src/test_runner2/TestResult.ts
@@ -0,0 +1,23 @@
+import TestStatus from './TestStatus';
+
+/**
+ * Indicates the result of a single test
+ */
+export interface TestResult {
+  /**
+   * The full human readable name of the test
+   */
+  name: string;
+  /**
+   * The status of the test
+   */
+  status: TestStatus;
+  /**
+   * The time it took to run the test
+   */
+  timeSpentMs: number;
+  /**
+   * Optional: message in case of status: Failed
+   */
+  failureMessage?: string;
+}
diff --git a/packages/api/src/test_runner2/TestRunner.ts b/packages/api/src/test_runner2/TestRunner.ts
new file mode 100644
index 0000000000..8e6f25dbeb
--- /dev/null
+++ b/packages/api/src/test_runner2/TestRunner.ts
@@ -0,0 +1,9 @@
+import { DryRunOptions, MutantRunOptions } from './RunOptions';
+import { MutantRunResult, DryRunResult } from './RunResult';
+
+export interface TestRunner2 {
+  init?(): Promise<void> | void;
+  dryRun(options: DryRunOptions): Promise<DryRunResult>;
+  mutantRun(options: MutantRunOptions): Promise<MutantRunResult>;
+  dispose?(): Promise<void> | void;
+}
diff --git a/packages/api/src/test_runner2/TestSelection.ts b/packages/api/src/test_runner2/TestSelection.ts
new file mode 100644
index 0000000000..2edf46c511
--- /dev/null
+++ b/packages/api/src/test_runner2/TestSelection.ts
@@ -0,0 +1,4 @@
+export default interface TestSelection {
+  id: number;
+  name: string;
+}
diff --git a/packages/api/src/test_runner2/TestStatus.ts b/packages/api/src/test_runner2/TestStatus.ts
new file mode 100644
index 0000000000..69bc8a823a
--- /dev/null
+++ b/packages/api/src/test_runner2/TestStatus.ts
@@ -0,0 +1,19 @@
+/**
+ * Indicates what the result of a single test was.
+ */
+enum TestStatus {
+  /**
+   * The test succeeded
+   */
+  Success,
+  /**
+   * The test failed
+   */
+  Failed,
+  /**
+   * The test was skipped (not executed)
+   */
+  Skipped,
+}
+
+export default TestStatus;
diff --git a/packages/api/test_runner2.ts b/packages/api/test_runner2.ts
new file mode 100644
index 0000000000..6e27f1fa2e
--- /dev/null
+++ b/packages/api/test_runner2.ts
@@ -0,0 +1,6 @@
+export * from './src/test_runner2/TestResult';
+export * from './src/test_runner2/TestRunner';
+export * from './src/test_runner2/TestStatus';
+export * from './src/test_runner2/RunResult';
+export * from './src/test_runner2/RunOptions';
+export * from './src/test_runner2/RunStatus';
diff --git a/packages/api/tsconfig.src.json b/packages/api/tsconfig.src.json
index b20098295e..7839a41cb9 100644
--- a/packages/api/tsconfig.src.json
+++ b/packages/api/tsconfig.src.json
@@ -14,6 +14,7 @@
     "report.ts",
     "test_framework.ts",
     "test_runner.ts",
+    "test_runner2.ts",
     "transpile.ts",
     "plugin.ts"
   ],
diff --git a/tsconfig.lint.json b/tsconfig.lint.json
index 91aa3ba6b6..f500c4bdde 100644
--- a/tsconfig.lint.json
+++ b/tsconfig.lint.json
@@ -1,9 +1,19 @@
 {
   "extends": "./tsconfig.settings.json",
-  // This file is a workaround for: https://github.com/palantir/tslint/issues/4137
+  // This file is a workaround for: https://github.com/typescript-eslint/typescript-eslint/tree/master/packages/parser#parseroptionsproject
   "include": [
     "packages/*/src",
     "packages/*/typings",
-    "packages/*/test"
+    "packages/*/test",
+    "packages/api/config.ts",
+    "packages/api/core.ts",
+    "packages/api/logging.ts",
+    "packages/api/mutant.ts",
+    "packages/api/report.ts",
+    "packages/api/test_framework.ts",
+    "packages/api/test_runner.ts",
+    "packages/api/test_runner2.ts",
+    "packages/api/transpile.ts",
+    "packages/api/plugin.ts"
   ]
 }

From 3b91826d96577b1f26cb6c118bfb31a702326cab Mon Sep 17 00:00:00 2001
From: Nico Jansen <jansennico@gmail.com>
Date: Tue, 9 Jun 2020 07:46:23 +0200
Subject: [PATCH 02/11] Add plugin type for test runner 2

---
 packages/api/src/plugin/Contexts.ts   | 1 +
 packages/api/src/plugin/PluginKind.ts | 1 +
 packages/api/src/plugin/Plugins.ts    | 2 ++
 3 files changed, 4 insertions(+)

diff --git a/packages/api/src/plugin/Contexts.ts b/packages/api/src/plugin/Contexts.ts
index b27e4a7f41..4973ce6753 100644
--- a/packages/api/src/plugin/Contexts.ts
+++ b/packages/api/src/plugin/Contexts.ts
@@ -47,5 +47,6 @@ export interface PluginContexts {
   [PluginKind.Reporter]: OptionsContext;
   [PluginKind.TestFramework]: OptionsContext;
   [PluginKind.TestRunner]: TestRunnerPluginContext;
+  [PluginKind.TestRunner2]: TestRunnerPluginContext;
   [PluginKind.Transpiler]: TranspilerPluginContext;
 }
diff --git a/packages/api/src/plugin/PluginKind.ts b/packages/api/src/plugin/PluginKind.ts
index d9e8133696..d312925764 100644
--- a/packages/api/src/plugin/PluginKind.ts
+++ b/packages/api/src/plugin/PluginKind.ts
@@ -8,6 +8,7 @@ export enum PluginKind {
   ConfigEditor = 'ConfigEditor',
   OptionsEditor = 'OptionsEditor',
   TestRunner = 'TestRunner',
+  TestRunner2 = 'TestRunner2',
   TestFramework = 'TestFramework',
   Transpiler = 'Transpiler',
   Mutator = 'Mutator',
diff --git a/packages/api/src/plugin/Plugins.ts b/packages/api/src/plugin/Plugins.ts
index 9b5e8f0dd7..c0f99d8bca 100644
--- a/packages/api/src/plugin/Plugins.ts
+++ b/packages/api/src/plugin/Plugins.ts
@@ -7,6 +7,7 @@ import { TestFramework } from '../../test_framework';
 import { TestRunner } from '../../test_runner';
 import { Transpiler } from '../../transpile';
 import { OptionsEditor } from '../core/OptionsEditor';
+import { TestRunner2 } from '../../test_runner2';
 
 import { PluginContexts } from './Contexts';
 import { PluginKind } from './PluginKind';
@@ -89,6 +90,7 @@ export interface PluginInterfaces {
   [PluginKind.Reporter]: Reporter;
   [PluginKind.TestFramework]: TestFramework;
   [PluginKind.TestRunner]: TestRunner;
+  [PluginKind.TestRunner2]: TestRunner2;
   [PluginKind.Transpiler]: Transpiler;
 }
 

From e495688579f862d7d0ebab3c201766b6f68ae7dc Mon Sep 17 00:00:00 2001
From: Nico Jansen <jansennico@gmail.com>
Date: Wed, 10 Jun 2020 19:05:11 +0200
Subject: [PATCH 03/11] fix: Import mutant from core

---
 packages/api/src/test_runner2/RunOptions.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/api/src/test_runner2/RunOptions.ts b/packages/api/src/test_runner2/RunOptions.ts
index 627725d410..259b7f778f 100644
--- a/packages/api/src/test_runner2/RunOptions.ts
+++ b/packages/api/src/test_runner2/RunOptions.ts
@@ -1,5 +1,5 @@
 import { TestSelection } from '../../test_framework';
-import { Mutant } from '../../mutant';
+import { Mutant } from '../../core';
 
 export interface RunOptions {
   /**

From 738493ac2c0499eb0d1b1720264d3c3c7f5e420a Mon Sep 17 00:00:00 2001
From: Nico Jansen <jansennico@gmail.com>
Date: Thu, 4 Jun 2020 20:12:16 +0200
Subject: [PATCH 04/11] refactor(util): move Task to @stryker-mutator/util

---
 .../core/src/child-proxy/ChildProcessProxy.ts |  3 +-
 .../ChildProcessTestRunnerDecorator.ts        |  4 +-
 .../core/src/test-runner/TimeoutDecorator.ts  |  7 +-
 packages/core/src/utils/objectUtils.ts        | 17 ----
 .../child-proxy/ChildProcessProxy.it.spec.ts  |  2 +-
 packages/core/test/unit/SandboxPool.spec.ts   |  2 +-
 .../ChildProcessTestRunnerDecorator.spec.ts   |  2 +-
 .../core/test/unit/utils/objectUtils.spec.ts  | 36 --------
 packages/core/tsconfig.src.json               |  2 +-
 packages/{core/src/utils => util/src}/Task.ts | 24 +++++-
 packages/util/src/index.ts                    |  1 +
 packages/util/test/unit/Task.spec.ts          | 86 +++++++++++++++++++
 12 files changed, 117 insertions(+), 69 deletions(-)
 delete mode 100644 packages/core/test/unit/utils/objectUtils.spec.ts
 rename packages/{core/src/utils => util/src}/Task.ts (53%)
 create mode 100644 packages/util/test/unit/Task.spec.ts

diff --git a/packages/core/src/child-proxy/ChildProcessProxy.ts b/packages/core/src/child-proxy/ChildProcessProxy.ts
index 096906d7fd..26551e971a 100644
--- a/packages/core/src/child-proxy/ChildProcessProxy.ts
+++ b/packages/core/src/child-proxy/ChildProcessProxy.ts
@@ -3,14 +3,13 @@ import * as os from 'os';
 
 import { File, StrykerOptions } from '@stryker-mutator/api/core';
 import { OptionsContext } from '@stryker-mutator/api/plugin';
-import { isErrnoException } from '@stryker-mutator/util';
+import { isErrnoException, Task, ExpirableTask } from '@stryker-mutator/util';
 import { getLogger } from 'log4js';
 import { Disposable, InjectableClass, InjectionToken } from 'typed-inject';
 
 import LoggingClientContext from '../logging/LoggingClientContext';
 import { deserialize, kill, padLeft, serialize } from '../utils/objectUtils';
 import StringBuilder from '../utils/StringBuilder';
-import { ExpirableTask, Task } from '../utils/Task';
 
 import ChildProcessCrashedError from './ChildProcessCrashedError';
 import { autoStart, ParentMessage, ParentMessageKind, WorkerMessage, WorkerMessageKind } from './messageProtocol';
diff --git a/packages/core/src/test-runner/ChildProcessTestRunnerDecorator.ts b/packages/core/src/test-runner/ChildProcessTestRunnerDecorator.ts
index b8e60306d2..a12cbe06f6 100644
--- a/packages/core/src/test-runner/ChildProcessTestRunnerDecorator.ts
+++ b/packages/core/src/test-runner/ChildProcessTestRunnerDecorator.ts
@@ -1,10 +1,10 @@
 import { StrykerOptions } from '@stryker-mutator/api/core';
 import { RunOptions, RunResult, TestRunner } from '@stryker-mutator/api/test_runner';
+import { ExpirableTask } from '@stryker-mutator/util';
 
 import ChildProcessCrashedError from '../child-proxy/ChildProcessCrashedError';
 import ChildProcessProxy from '../child-proxy/ChildProcessProxy';
 import LoggingClientContext from '../logging/LoggingClientContext';
-import { timeout } from '../utils/objectUtils';
 
 import { ChildProcessTestRunnerWorker } from './ChildProcessTestRunnerWorker';
 
@@ -36,7 +36,7 @@ export default class ChildProcessTestRunnerDecorator implements TestRunner {
   }
 
   public async dispose(): Promise<void> {
-    await timeout(
+    await ExpirableTask.timeout(
       // First let the inner test runner dispose
       this.worker.proxy.dispose().catch((error) => {
         // It's OK if the child process is already down.
diff --git a/packages/core/src/test-runner/TimeoutDecorator.ts b/packages/core/src/test-runner/TimeoutDecorator.ts
index f6c38e49ca..bdeb801fd3 100644
--- a/packages/core/src/test-runner/TimeoutDecorator.ts
+++ b/packages/core/src/test-runner/TimeoutDecorator.ts
@@ -1,7 +1,6 @@
 import { RunOptions, RunResult, RunStatus } from '@stryker-mutator/api/test_runner';
 import { getLogger } from 'log4js';
-
-import { timeout, TimeoutExpired } from '../utils/objectUtils';
+import { ExpirableTask } from '@stryker-mutator/util';
 
 import TestRunnerDecorator from './TestRunnerDecorator';
 
@@ -13,8 +12,8 @@ export default class TimeoutDecorator extends TestRunnerDecorator {
 
   public async run(options: RunOptions): Promise<RunResult> {
     this.log.debug('Starting timeout timer (%s ms) for a test run', options.timeout);
-    const result = await timeout(super.run(options), options.timeout);
-    if (result === TimeoutExpired) {
+    const result = await ExpirableTask.timeout(super.run(options), options.timeout);
+    if (result === ExpirableTask.TimeoutExpired) {
       return this.handleTimeout();
     } else {
       return result;
diff --git a/packages/core/src/utils/objectUtils.ts b/packages/core/src/utils/objectUtils.ts
index 3d98cc8caf..c7155ab266 100644
--- a/packages/core/src/utils/objectUtils.ts
+++ b/packages/core/src/utils/objectUtils.ts
@@ -64,23 +64,6 @@ export function kill(pid: number): Promise<void> {
   });
 }
 
-export const TimeoutExpired: unique symbol = Symbol('TimeoutExpired');
-export function timeout<T>(promise: Promise<T>, ms: number): Promise<T | typeof TimeoutExpired> {
-  const sleep = new Promise<T | typeof TimeoutExpired>((res, rej) => {
-    const timer = setTimeout(() => res(TimeoutExpired), ms);
-    promise
-      .then((result) => {
-        clearTimeout(timer);
-        res(result);
-      })
-      .catch((error) => {
-        clearTimeout(timer);
-        rej(error);
-      });
-  });
-  return sleep;
-}
-
 export function padLeft(input: string): string {
   return input
     .split('\n')
diff --git a/packages/core/test/integration/child-proxy/ChildProcessProxy.it.spec.ts b/packages/core/test/integration/child-proxy/ChildProcessProxy.it.spec.ts
index 9c5b0241c3..821cb3fa70 100644
--- a/packages/core/test/integration/child-proxy/ChildProcessProxy.it.spec.ts
+++ b/packages/core/test/integration/child-proxy/ChildProcessProxy.it.spec.ts
@@ -7,11 +7,11 @@ import { testInjector, LoggingServer } from '@stryker-mutator/test-helpers';
 import { expect } from 'chai';
 import * as log4js from 'log4js';
 import { filter } from 'rxjs/operators';
+import { Task } from '@stryker-mutator/util';
 
 import ChildProcessCrashedError from '../../../src/child-proxy/ChildProcessCrashedError';
 import ChildProcessProxy from '../../../src/child-proxy/ChildProcessProxy';
 import OutOfMemoryError from '../../../src/child-proxy/OutOfMemoryError';
-import { Task } from '../../../src/utils/Task';
 import currentLogMock from '../../helpers/logMock';
 import { Mock } from '../../helpers/producers';
 import { sleep } from '../../helpers/testUtils';
diff --git a/packages/core/test/unit/SandboxPool.spec.ts b/packages/core/test/unit/SandboxPool.spec.ts
index 4d5a77cb85..060854bf9e 100644
--- a/packages/core/test/unit/SandboxPool.spec.ts
+++ b/packages/core/test/unit/SandboxPool.spec.ts
@@ -11,6 +11,7 @@ import { expect } from 'chai';
 import { from } from 'rxjs';
 import { toArray } from 'rxjs/operators';
 import * as sinon from 'sinon';
+import { Task } from '@stryker-mutator/util';
 
 import { coreTokens } from '../../src/di';
 import LoggingClientContext from '../../src/logging/LoggingClientContext';
@@ -18,7 +19,6 @@ import { InitialTestRunResult } from '../../src/process/InitialTestExecutor';
 import Sandbox from '../../src/Sandbox';
 import { SandboxPool } from '../../src/SandboxPool';
 import TranspiledMutant from '../../src/TranspiledMutant';
-import { Task } from '../../src/utils/Task';
 import { TemporaryDirectory } from '../../src/utils/TemporaryDirectory';
 import { Mock, mock, transpiledMutant } from '../helpers/producers';
 
diff --git a/packages/core/test/unit/test-runner/ChildProcessTestRunnerDecorator.spec.ts b/packages/core/test/unit/test-runner/ChildProcessTestRunnerDecorator.spec.ts
index fe39b8a9f1..4b3615fb4f 100644
--- a/packages/core/test/unit/test-runner/ChildProcessTestRunnerDecorator.spec.ts
+++ b/packages/core/test/unit/test-runner/ChildProcessTestRunnerDecorator.spec.ts
@@ -3,6 +3,7 @@ import { RunOptions } from '@stryker-mutator/api/test_runner';
 import { strykerOptions } from '@stryker-mutator/test-helpers/src/factory';
 import { expect } from 'chai';
 import * as sinon from 'sinon';
+import { Task } from '@stryker-mutator/util';
 
 import ChildProcessCrashedError from '../../../src/child-proxy/ChildProcessCrashedError';
 import ChildProcessProxy from '../../../src/child-proxy/ChildProcessProxy';
@@ -10,7 +11,6 @@ import LoggingClientContext from '../../../src/logging/LoggingClientContext';
 import ChildProcessTestRunnerDecorator from '../../../src/test-runner/ChildProcessTestRunnerDecorator';
 import { ChildProcessTestRunnerWorker } from '../../../src/test-runner/ChildProcessTestRunnerWorker';
 import TestRunnerDecorator from '../../../src/test-runner/TestRunnerDecorator';
-import { Task } from '../../../src/utils/Task';
 import { Mock, mock } from '../../helpers/producers';
 
 describe(ChildProcessTestRunnerDecorator.name, () => {
diff --git a/packages/core/test/unit/utils/objectUtils.spec.ts b/packages/core/test/unit/utils/objectUtils.spec.ts
deleted file mode 100644
index b19267f9a5..0000000000
--- a/packages/core/test/unit/utils/objectUtils.spec.ts
+++ /dev/null
@@ -1,36 +0,0 @@
-import { expect } from 'chai';
-import { match } from 'sinon';
-import * as sinon from 'sinon';
-
-import * as sut from '../../../src/utils/objectUtils';
-import { Task } from '../../../src/utils/Task';
-
-describe('objectUtils', () => {
-  describe('timeout', () => {
-    it('should timeout a promise after a set period', async () => {
-      const task = new Task();
-      const actual = await sut.timeout(task.promise, 0);
-      expect(actual).eq(sut.TimeoutExpired);
-      task.resolve(undefined);
-    });
-
-    it('should remove any nodejs timers when promise resolves', async () => {
-      // Arrange
-      const expectedTimer = 234;
-      const setTimeoutStub = sinon.stub(global, 'setTimeout');
-      const clearTimeoutStub = sinon.stub(global, 'clearTimeout');
-      setTimeoutStub.returns(expectedTimer);
-      const expectedResult = 'expectedResult';
-      const p = Promise.resolve(expectedResult);
-      const delay = 10;
-
-      // Act
-      const result = await sut.timeout(p, delay);
-
-      // Assert
-      expect(result).eq(expectedResult);
-      expect(clearTimeoutStub).calledWith(expectedTimer);
-      expect(setTimeoutStub).calledWith(match.func, delay);
-    });
-  });
-});
diff --git a/packages/core/tsconfig.src.json b/packages/core/tsconfig.src.json
index 79345d075c..ea4f5866dd 100644
--- a/packages/core/tsconfig.src.json
+++ b/packages/core/tsconfig.src.json
@@ -6,7 +6,7 @@
     "types": [] // Exclude global mocha functions for the sources
   },
   "include": [
-    "src"
+    "src", "../util/src/Task.ts"
   ],
   "references": [
     {
diff --git a/packages/core/src/utils/Task.ts b/packages/util/src/Task.ts
similarity index 53%
rename from packages/core/src/utils/Task.ts
rename to packages/util/src/Task.ts
index 69cfd7c268..4a8bddcfd3 100644
--- a/packages/core/src/utils/Task.ts
+++ b/packages/util/src/Task.ts
@@ -1,5 +1,3 @@
-import { timeout, TimeoutExpired } from './objectUtils';
-
 /**
  * Wraps a promise in a Task api for convenience.
  */
@@ -38,9 +36,27 @@ export class Task<T = void> {
 /**
  * A task that can expire after the given time.
  */
-export class ExpirableTask<T = void> extends Task<T | typeof TimeoutExpired> {
+export class ExpirableTask<T = void> extends Task<T | typeof ExpirableTask.TimeoutExpired> {
+  public static readonly TimeoutExpired: unique symbol = Symbol('TimeoutExpired');
+
   constructor(timeoutMS: number) {
     super();
-    this._promise = timeout(this._promise, timeoutMS);
+    this._promise = ExpirableTask.timeout(this._promise, timeoutMS);
+  }
+
+  public static timeout<T>(promise: Promise<T>, ms: number): Promise<T | typeof ExpirableTask.TimeoutExpired> {
+    const sleep = new Promise<T | typeof ExpirableTask.TimeoutExpired>((res, rej) => {
+      const timer = setTimeout(() => res(ExpirableTask.TimeoutExpired), ms);
+      promise
+        .then((result) => {
+          clearTimeout(timer);
+          res(result);
+        })
+        .catch((error) => {
+          clearTimeout(timer);
+          rej(error);
+        });
+    });
+    return sleep;
   }
 }
diff --git a/packages/util/src/index.ts b/packages/util/src/index.ts
index 064f2faadf..4950857c2a 100644
--- a/packages/util/src/index.ts
+++ b/packages/util/src/index.ts
@@ -8,3 +8,4 @@ export * from './noopLogger';
 export * from './notEmpty';
 export * from './flatMap';
 export * from './I';
+export * from './Task';
diff --git a/packages/util/test/unit/Task.spec.ts b/packages/util/test/unit/Task.spec.ts
new file mode 100644
index 0000000000..dd78a1538e
--- /dev/null
+++ b/packages/util/test/unit/Task.spec.ts
@@ -0,0 +1,86 @@
+import { expect } from 'chai';
+
+import { Task, ExpirableTask } from '../../src';
+
+import sinon = require('sinon');
+
+describe(Task.name, () => {
+  it('should give access to underlying promise', () => {
+    const sut = new Task();
+    expect(sut.promise).instanceOf(Promise);
+    sut.resolve();
+  });
+
+  it('should be able to resolve the underlying promise', async () => {
+    const sut = new Task<string>();
+    sut.resolve('resolved');
+    const result = await sut.promise;
+    expect(result).eq('resolved');
+  });
+
+  it('should be able to reject the underlying promise', async () => {
+    const sut = new Task<string>();
+    const expectedError = new Error('expected error');
+    sut.reject(expectedError);
+    await expect(sut.promise).rejectedWith(expectedError);
+  });
+
+  it('should be able to know if it isCompleted', () => {
+    const sut = new Task();
+    expect(sut.isCompleted).false;
+    sut.resolve();
+    expect(sut.isCompleted).true;
+  });
+});
+
+describe(ExpirableTask.name, () => {
+  describe('instance', () => {
+    it('should timeout after set period', async () => {
+      const task = new ExpirableTask(0);
+      const result = await task.promise;
+      expect(result).eq(ExpirableTask.TimeoutExpired);
+    });
+
+    it('should be able to resolve within time', async () => {
+      const task = new ExpirableTask<string>(0);
+      task.resolve('in time');
+      const result = await task.promise;
+      expect(result).eq('in time');
+    });
+
+    it('should be able to reject within time', async () => {
+      const task = new ExpirableTask(0);
+      const expectedError = new Error('expected error');
+      task.reject(expectedError);
+      await expect(task.promise).rejectedWith(expectedError);
+    });
+  });
+
+  describe('timeout', () => {
+    it('should timeout a promise after a set period', async () => {
+      const task = new Task();
+      const actual = await ExpirableTask.timeout(task.promise, 0);
+      expect(actual).eq(ExpirableTask.TimeoutExpired);
+      task.resolve(undefined);
+    });
+
+    it('should remove any nodejs timers when promise resolves', async () => {
+      // Arrange
+      const expectedTimer = 234;
+      const setTimeoutStub = sinon.stub(global, 'setTimeout');
+      const clearTimeoutStub = sinon.stub(global, 'clearTimeout');
+      setTimeoutStub.returns(expectedTimer);
+      const expectedResult = 'expectedResult';
+      const p = Promise.resolve(expectedResult);
+      const delay = 10;
+
+      // Act
+      const result = await ExpirableTask.timeout(p, delay);
+
+      // Assert
+      expect(result).eq(expectedResult);
+      expect(clearTimeoutStub).calledWith(expectedTimer);
+      expect(setTimeoutStub).calledWith(sinon.match.func, delay);
+    });
+  });
+});

From 7c575b888ca90fffc0bf8285e1d452d363bba150 Mon Sep 17 00:00:00 2001
From: Nico Jansen <jansennico@gmail.com>
Date: Wed, 10 Jun 2020 23:04:31 +0200
Subject: [PATCH 05/11] Remove error messages array

---
 packages/api/src/test_runner2/RunResult.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/api/src/test_runner2/RunResult.ts b/packages/api/src/test_runner2/RunResult.ts
index fb4170f966..a2a72877c3 100644
--- a/packages/api/src/test_runner2/RunResult.ts
+++ b/packages/api/src/test_runner2/RunResult.ts
@@ -19,7 +19,7 @@ export interface RunResult {
   /**
    * If `state` is `error`, this collection should contain the error messages
    */
-  errorMessages?: string[];
+  errorMessage?: string;
 
   /**
    * The status of the run

From bc44350e0161332aa9d449db87c20602a149a9f7 Mon Sep 17 00:00:00 2001
From: Nico Jansen <jansennico@gmail.com>
Date: Thu, 11 Jun 2020 18:06:09 +0200
Subject: [PATCH 06/11] Add new suggestion for test runner api

---
 packages/api/src/test_runner2/RunResult.ts | 38 +++++++++++++++-------
 1 file changed, 26 insertions(+), 12 deletions(-)

diff --git a/packages/api/src/test_runner2/RunResult.ts b/packages/api/src/test_runner2/RunResult.ts
index a2a72877c3..978b1c24bc 100644
--- a/packages/api/src/test_runner2/RunResult.ts
+++ b/packages/api/src/test_runner2/RunResult.ts
@@ -1,32 +1,46 @@
+import { TestSelection } from '../../test_framework';
+
 import { MutantCoverage } from './MutantCoverage';
 import { RunStatus } from './RunStatus';
 import { TestResult } from './TestResult';
 
-/**
- * Marker interface for a mutant run result.
- */
-export interface MutantRunResult extends RunResult {}
+export enum MutantRunStatus {
+  Killed = 'killed',
+  Survived = 'survived',
+}
+
+export type MutantRunResult = KilledMutantRunResult | SurvivedMutantRunResult;
+
+export interface KilledMutantRunResult extends RunResult {
+  status: MutantRunStatus.Killed;
+  culpritTest: TestSelection;
+}
+
+export interface SurvivedMutantRunResult extends RunResult {
+  status: MutantRunStatus.Survived;
+  testSelection: TestSelection[] | false;
+}
 
 /**
  * Represents the result of a test run.
  */
 export interface RunResult {
   /**
-   * The individual test results.
+   * If `state` is `error`, this collection should contain the error messages
    */
-  tests: TestResult[];
+  errorMessage?: string;
+}
 
+export interface DryRunResult extends RunResult {
   /**
-   * If `state` is `error`, this collection should contain the error messages
+   * The individual test results.
    */
-  errorMessage?: string;
+  tests: TestResult[];
+
+  mutationCoverage?: MutantCoverage;
 
   /**
    * The status of the run
    */
   status: RunStatus;
 }
-
-export interface DryRunResult extends RunResult {
-  mutationCoverage?: MutantCoverage;
-}

From a44409bb7f92f2747a5bf3010ac62b7fe6833167 Mon Sep 17 00:00:00 2001
From: Nico Jansen <jansennico@gmail.com>
Date: Thu, 11 Jun 2020 19:34:09 +0200
Subject: [PATCH 07/11] Review test runner api

---
 .../api/src/test_runner2/MutantCoverage.ts    |  5 +-
 packages/api/src/test_runner2/RunOptions.ts   |  2 +-
 packages/api/src/test_runner2/RunResult.ts    | 52 ++++++++++++++-----
 3 files changed, 41 insertions(+), 18 deletions(-)

diff --git a/packages/api/src/test_runner2/MutantCoverage.ts b/packages/api/src/test_runner2/MutantCoverage.ts
index 8298a0856f..6a8b644329 100644
--- a/packages/api/src/test_runner2/MutantCoverage.ts
+++ b/packages/api/src/test_runner2/MutantCoverage.ts
@@ -1,7 +1,6 @@
-export type MutantCoverage = StaticCoverage & CoveragePerTestId;
-
-export interface StaticCoverage {
+export interface MutantCoverage {
   static: CoverageData;
+  perTest: CoveragePerTestId;
 }
 
 export interface CoveragePerTestId {
diff --git a/packages/api/src/test_runner2/RunOptions.ts b/packages/api/src/test_runner2/RunOptions.ts
index 259b7f778f..59782defc1 100644
--- a/packages/api/src/test_runner2/RunOptions.ts
+++ b/packages/api/src/test_runner2/RunOptions.ts
@@ -12,7 +12,7 @@ export interface DryRunOptions extends RunOptions {
   /**
    * Indicates whether or not mutant coverage should be collected.
    */
-  reportMutantCoverage: boolean;
+  coverageAnalysis: 'off' | 'all' | 'perTest';
 }
 
 export interface MutantRunOptions extends RunOptions {
diff --git a/packages/api/src/test_runner2/RunResult.ts b/packages/api/src/test_runner2/RunResult.ts
index 978b1c24bc..73f32aef50 100644
--- a/packages/api/src/test_runner2/RunResult.ts
+++ b/packages/api/src/test_runner2/RunResult.ts
@@ -7,31 +7,37 @@ import { TestResult } from './TestResult';
 export enum MutantRunStatus {
   Killed = 'killed',
   Survived = 'survived',
+  TimedOut = 'timedOut',
+  Error = 'error',
 }
 
 export type MutantRunResult = KilledMutantRunResult | SurvivedMutantRunResult;
 
-export interface KilledMutantRunResult extends RunResult {
+export interface TimedOutMutantRunResult {
+  status: MutantRunStatus.TimedOut;
+}
+
+export interface TimedOutMutantRunResult {
+  status: MutantRunStatus.TimedOut;
+}
+
+export interface KilledMutantRunResult {
   status: MutantRunStatus.Killed;
-  culpritTest: TestSelection;
+  killedBy: TestSelection;
 }
 
-export interface SurvivedMutantRunResult extends RunResult {
+export interface SurvivedMutantRunResult {
   status: MutantRunStatus.Survived;
-  testSelection: TestSelection[] | false;
 }
 
-/**
- * Represents the result of a test run.
- */
-export interface RunResult {
-  /**
-   * If `state` is `error`, this collection should contain the error messages
-   */
-  errorMessage?: string;
+export interface ErrorMutantRunResult {
+  status: MutantRunStatus.Error;
+  errorMessage: string;
 }
 
-export interface DryRunResult extends RunResult {
+export type DryRunResult = CompleteDryRunResult | TimeoutDryRunResult | ErrorDryRunResult;
+
+export interface CompleteDryRunResult {
   /**
    * The individual test results.
    */
@@ -42,5 +48,23 @@ export interface DryRunResult extends RunResult {
   /**
    * The status of the run
    */
-  status: RunStatus;
+  status: RunStatus.Complete;
+}
+export interface TimeoutDryRunResult {
+  /**
+   * The status of the run
+   */
+  status: RunStatus.Timeout;
+}
+
+export interface ErrorDryRunResult {
+  /**
+   * The status of the run
+   */
+  status: RunStatus.Error;
+
+  /**
+   * If `state` is `error`, this collection should contain the error messages
+   */
+  errorMessage: string;
 }

From cff2f37e2241791b1dcd8123f51330b533103391 Mon Sep 17 00:00:00 2001
From: Nico Jansen <jansennico@gmail.com>
Date: Thu, 11 Jun 2020 20:30:11 +0200
Subject: [PATCH 08/11] feat(api): further cleanup test runner api

---
 .../{RunResult.ts => DryRunResult.ts}         | 33 -------------------
 .../api/src/test_runner2/MutantRunResult.ts   | 29 ++++++++++++++++
 packages/api/src/test_runner2/RunOptions.ts   |  3 +-
 packages/api/src/test_runner2/TestRunner.ts   |  3 +-
 .../api/src/test_runner2/TestSelection.ts     |  2 +-
 packages/api/test_runner2.ts                  |  4 ++-
 6 files changed, 37 insertions(+), 37 deletions(-)
 rename packages/api/src/test_runner2/{RunResult.ts => DryRunResult.ts} (52%)
 create mode 100644 packages/api/src/test_runner2/MutantRunResult.ts

diff --git a/packages/api/src/test_runner2/RunResult.ts b/packages/api/src/test_runner2/DryRunResult.ts
similarity index 52%
rename from packages/api/src/test_runner2/RunResult.ts
rename to packages/api/src/test_runner2/DryRunResult.ts
index 73f32aef50..c200b0a066 100644
--- a/packages/api/src/test_runner2/RunResult.ts
+++ b/packages/api/src/test_runner2/DryRunResult.ts
@@ -1,40 +1,7 @@
-import { TestSelection } from '../../test_framework';
-
 import { MutantCoverage } from './MutantCoverage';
 import { RunStatus } from './RunStatus';
 import { TestResult } from './TestResult';
 
-export enum MutantRunStatus {
-  Killed = 'killed',
-  Survived = 'survived',
-  TimedOut = 'timedOut',
-  Error = 'error',
-}
-
-export type MutantRunResult = KilledMutantRunResult | SurvivedMutantRunResult;
-
-export interface TimedOutMutantRunResult {
-  status: MutantRunStatus.TimedOut;
-}
-
-export interface TimedOutMutantRunResult {
-  status: MutantRunStatus.TimedOut;
-}
-
-export interface KilledMutantRunResult {
-  status: MutantRunStatus.Killed;
-  killedBy: TestSelection;
-}
-
-export interface SurvivedMutantRunResult {
-  status: MutantRunStatus.Survived;
-}
-
-export interface ErrorMutantRunResult {
-  status: MutantRunStatus.Error;
-  errorMessage: string;
-}
-
 export type DryRunResult = CompleteDryRunResult | TimeoutDryRunResult | ErrorDryRunResult;
 
 export interface CompleteDryRunResult {
diff --git a/packages/api/src/test_runner2/MutantRunResult.ts b/packages/api/src/test_runner2/MutantRunResult.ts
new file mode 100644
index 0000000000..85b72582f5
--- /dev/null
+++ b/packages/api/src/test_runner2/MutantRunResult.ts
@@ -0,0 +1,29 @@
+import { TestSelection } from './TestSelection';
+
+export enum MutantRunStatus {
+  Killed = 'killed',
+  Survived = 'survived',
+  Timeout = 'timeout',
+  Error = 'error',
+}
+
+export type MutantRunResult = KilledMutantRunResult | SurvivedMutantRunResult | TimeoutMutantRunResult | ErrorMutantRunResult;
+
+export interface TimeoutMutantRunResult {
+  status: MutantRunStatus.Timeout;
+}
+
+export interface KilledMutantRunResult {
+  status: MutantRunStatus.Killed;
+  killedBy: TestSelection;
+  failureMessage: string;
+}
+
+export interface SurvivedMutantRunResult {
+  status: MutantRunStatus.Survived;
+}
+
+export interface ErrorMutantRunResult {
+  status: MutantRunStatus.Error;
+  errorMessage: string;
+}
diff --git a/packages/api/src/test_runner2/RunOptions.ts b/packages/api/src/test_runner2/RunOptions.ts
index 59782defc1..a8597ee1be 100644
--- a/packages/api/src/test_runner2/RunOptions.ts
+++ b/packages/api/src/test_runner2/RunOptions.ts
@@ -1,6 +1,7 @@
-import { TestSelection } from '../../test_framework';
 import { Mutant } from '../../core';
 
+import { TestSelection } from './TestSelection';
+
 export interface RunOptions {
   /**
    * The amount of time (in milliseconds) the TestRunner has to complete the test run before a timeout occurs.
diff --git a/packages/api/src/test_runner2/TestRunner.ts b/packages/api/src/test_runner2/TestRunner.ts
index 8e6f25dbeb..6427bb859a 100644
--- a/packages/api/src/test_runner2/TestRunner.ts
+++ b/packages/api/src/test_runner2/TestRunner.ts
@@ -1,5 +1,6 @@
 import { DryRunOptions, MutantRunOptions } from './RunOptions';
-import { MutantRunResult, DryRunResult } from './RunResult';
+import { DryRunResult } from './DryRunResult';
+import { MutantRunResult } from './MutantRunResult';
 
 export interface TestRunner2 {
   init?(): Promise<void> | void;
diff --git a/packages/api/src/test_runner2/TestSelection.ts b/packages/api/src/test_runner2/TestSelection.ts
index 2edf46c511..dd0d8cddf4 100644
--- a/packages/api/src/test_runner2/TestSelection.ts
+++ b/packages/api/src/test_runner2/TestSelection.ts
@@ -1,4 +1,4 @@
-export default interface TestSelection {
+export interface TestSelection {
   id: number;
   name: string;
 }
diff --git a/packages/api/test_runner2.ts b/packages/api/test_runner2.ts
index 6e27f1fa2e..b62eee09af 100644
--- a/packages/api/test_runner2.ts
+++ b/packages/api/test_runner2.ts
@@ -1,6 +1,8 @@
 export * from './src/test_runner2/TestResult';
 export * from './src/test_runner2/TestRunner';
 export * from './src/test_runner2/TestStatus';
-export * from './src/test_runner2/RunResult';
+export * from './src/test_runner2/DryRunResult';
 export * from './src/test_runner2/RunOptions';
+export * from './src/test_runner2/MutantRunResult';
+export * from './src/test_runner2/TestSelection';
 export * from './src/test_runner2/RunStatus';

From e70eec67bf952980ad5a478411d6b9d07bce5c4d Mon Sep 17 00:00:00 2001
From: Nico Jansen <jansennico@gmail.com>
Date: Thu, 11 Jun 2020 20:52:43 +0200
Subject: [PATCH 09/11] Add runResult helper to convert a dry run result to a
 mutant run result

---
 .../api/src/test_runner2/MutantCoverage.ts    |  2 +-
 .../api/src/test_runner2/MutantRunResult.ts   | 10 ++++--
 packages/api/src/test_runner2/RunOptions.ts   |  4 +--
 packages/api/src/test_runner2/RunStatus.ts    |  6 ++--
 packages/api/src/test_runner2/TestResult.ts   | 29 +++++++++++-----
 .../api/src/test_runner2/TestSelection.ts     |  4 ---
 .../api/src/test_runner2/runResultHelpers.ts  | 34 +++++++++++++++++++
 packages/api/test_runner2.ts                  |  2 +-
 8 files changed, 67 insertions(+), 24 deletions(-)
 delete mode 100644 packages/api/src/test_runner2/TestSelection.ts
 create mode 100644 packages/api/src/test_runner2/runResultHelpers.ts

diff --git a/packages/api/src/test_runner2/MutantCoverage.ts b/packages/api/src/test_runner2/MutantCoverage.ts
index 6a8b644329..2170c27be9 100644
--- a/packages/api/src/test_runner2/MutantCoverage.ts
+++ b/packages/api/src/test_runner2/MutantCoverage.ts
@@ -4,7 +4,7 @@ export interface MutantCoverage {
 }
 
 export interface CoveragePerTestId {
-  [testId: number]: CoverageData;
+  [testId: string]: CoverageData;
 }
 
 export interface CoverageData {
diff --git a/packages/api/src/test_runner2/MutantRunResult.ts b/packages/api/src/test_runner2/MutantRunResult.ts
index 85b72582f5..3713972741 100644
--- a/packages/api/src/test_runner2/MutantRunResult.ts
+++ b/packages/api/src/test_runner2/MutantRunResult.ts
@@ -1,5 +1,3 @@
-import { TestSelection } from './TestSelection';
-
 export enum MutantRunStatus {
   Killed = 'killed',
   Survived = 'survived',
@@ -15,7 +13,13 @@ export interface TimeoutMutantRunResult {
 
 export interface KilledMutantRunResult {
   status: MutantRunStatus.Killed;
-  killedBy: TestSelection;
+  /**
+   * The id of the test that killed this mutant
+   */
+  killedBy: string;
+  /**
+   * The failure message that was reported by the test
+   */
   failureMessage: string;
 }
 
diff --git a/packages/api/src/test_runner2/RunOptions.ts b/packages/api/src/test_runner2/RunOptions.ts
index a8597ee1be..ac390917af 100644
--- a/packages/api/src/test_runner2/RunOptions.ts
+++ b/packages/api/src/test_runner2/RunOptions.ts
@@ -1,7 +1,5 @@
 import { Mutant } from '../../core';
 
-import { TestSelection } from './TestSelection';
-
 export interface RunOptions {
   /**
    * The amount of time (in milliseconds) the TestRunner has to complete the test run before a timeout occurs.
@@ -17,7 +15,7 @@ export interface DryRunOptions extends RunOptions {
 }
 
 export interface MutantRunOptions extends RunOptions {
-  testFilter?: TestSelection[];
+  testFilter?: string[];
   activeMutant: Mutant;
 }
 
diff --git a/packages/api/src/test_runner2/RunStatus.ts b/packages/api/src/test_runner2/RunStatus.ts
index 86fe77af14..b9d8e92b4d 100644
--- a/packages/api/src/test_runner2/RunStatus.ts
+++ b/packages/api/src/test_runner2/RunStatus.ts
@@ -2,13 +2,13 @@ export enum RunStatus {
   /**
    * Indicates that a test run is completed with failed or succeeded tests
    */
-  Complete,
+  Complete = 'complete',
   /**
    * Indicates that a test run cut off early with an error
    */
-  Error,
+  Error = 'error',
   /**
    * Indicates that a test run timed out
    */
-  Timeout,
+  Timeout = 'timeout',
 }
diff --git a/packages/api/src/test_runner2/TestResult.ts b/packages/api/src/test_runner2/TestResult.ts
index 35936863fa..7b4322a1ae 100644
--- a/packages/api/src/test_runner2/TestResult.ts
+++ b/packages/api/src/test_runner2/TestResult.ts
@@ -3,21 +3,32 @@ import TestStatus from './TestStatus';
 /**
  * Indicates the result of a single test
  */
-export interface TestResult {
+interface BaseTestResult {
   /**
-   * The full human readable name of the test
+   * The id of this test. Can be the name if the test runner doesn't have an 'id'
    */
-  name: string;
+  id: string;
   /**
-   * The status of the test
+   * The full human readable name of the test
    */
-  status: TestStatus;
+  name: string;
   /**
    * The time it took to run the test
    */
   timeSpentMs: number;
-  /**
-   * Optional: message in case of status: Failed
-   */
-  failureMessage?: string;
 }
+
+export interface FailedTestResult extends BaseTestResult {
+  status: TestStatus.Failed;
+  failureMessage: string;
+}
+
+export interface SkippedTestResult extends BaseTestResult {
+  status: TestStatus.Skipped;
+}
+
+export interface SuccessTestResult extends BaseTestResult {
+  status: TestStatus.Success;
+}
+
+export type TestResult = SuccessTestResult | FailedTestResult | SkippedTestResult;
diff --git a/packages/api/src/test_runner2/TestSelection.ts b/packages/api/src/test_runner2/TestSelection.ts
deleted file mode 100644
index dd0d8cddf4..0000000000
--- a/packages/api/src/test_runner2/TestSelection.ts
+++ /dev/null
@@ -1,4 +0,0 @@
-export interface TestSelection {
-  id: number;
-  name: string;
-}
diff --git a/packages/api/src/test_runner2/runResultHelpers.ts b/packages/api/src/test_runner2/runResultHelpers.ts
new file mode 100644
index 0000000000..4955186055
--- /dev/null
+++ b/packages/api/src/test_runner2/runResultHelpers.ts
@@ -0,0 +1,34 @@
+import TestStatus from './TestStatus';
+import { DryRunResult } from './DryRunResult';
+import { MutantRunResult } from './MutantRunResult';
+import { RunStatus } from './RunStatus';
+import { MutantRunStatus } from './MutantRunResult';
+import { FailedTestResult } from './TestResult';
+
+export function toMutantRunResult(dryRunResult: DryRunResult): MutantRunResult {
+  switch (dryRunResult.status) {
+    case RunStatus.Complete: {
+      const killedBy = dryRunResult.tests.find<FailedTestResult>((test): test is FailedTestResult => test.status === TestStatus.Failed);
+      if (killedBy) {
+        return {
+          status: MutantRunStatus.Killed,
+          failureMessage: killedBy.failureMessage,
+          killedBy: killedBy.id,
+        };
+      } else {
+        return {
+          status: MutantRunStatus.Survived,
+        };
+      }
+    }
+    case RunStatus.Error:
+      return {
+        status: MutantRunStatus.Error,
+        errorMessage: dryRunResult.errorMessage,
+      };
+    case RunStatus.Timeout:
+      return {
+        status: MutantRunStatus.Timeout,
+      };
+  }
+}
diff --git a/packages/api/test_runner2.ts b/packages/api/test_runner2.ts
index b62eee09af..4b9dd1c95a 100644
--- a/packages/api/test_runner2.ts
+++ b/packages/api/test_runner2.ts
@@ -4,5 +4,5 @@ export * from './src/test_runner2/TestStatus';
 export * from './src/test_runner2/DryRunResult';
 export * from './src/test_runner2/RunOptions';
 export * from './src/test_runner2/MutantRunResult';
-export * from './src/test_runner2/TestSelection';
 export * from './src/test_runner2/RunStatus';
+export * from './src/test_runner2/runResultHelpers';

From 1ec54db4282191d2958a6a9e57f8cfc4fa2795a7 Mon Sep 17 00:00:00 2001
From: Nico Jansen <jansennico@gmail.com>
Date: Thu, 11 Jun 2020 21:14:04 +0200
Subject: [PATCH 10/11] Add tests for toMutantRunResult

---
 packages/api/.vscode/launch.json              | 23 ++------
 .../test_runner2/runResultHelpers.spec.ts     | 56 +++++++++++++++++++
 2 files changed, 60 insertions(+), 19 deletions(-)
 create mode 100644 packages/api/test/unit/test_runner2/runResultHelpers.spec.ts

diff --git a/packages/api/.vscode/launch.json b/packages/api/.vscode/launch.json
index 090eaac215..c7285fbf35 100644
--- a/packages/api/.vscode/launch.json
+++ b/packages/api/.vscode/launch.json
@@ -1,24 +1,6 @@
 {
   "version": "0.2.0",
   "configurations": [
-    {
-      "type": "node",
-      "request": "launch",
-      "name": "Integration tests",
-      "program": "${workspaceFolder}/../../node_modules/mocha/bin/_mocha",
-      "args": [
-        "--timeout",
-        "999999",
-        "--colors",
-        "${workspaceFolder}/test/helpers/**/*.js",
-        "${workspaceFolder}/test/integration/**/*.js"
-      ],
-      "internalConsoleOptions": "openOnSessionStart",
-      "outFiles": [
-        "${workspaceRoot}/test/**/*.js",
-        "${workspaceRoot}/src/**/*.js"
-      ]
-    },
     {
       "type": "node",
       "request": "launch",
@@ -35,7 +17,10 @@
       "outFiles": [
         "${workspaceRoot}/test/**/*.js",
         "${workspaceRoot}/src/**/*.js"
+      ],
+      "skipFiles": [
+        "<node_internals>/**"
       ]
     }
   ]
-}
\ No newline at end of file
+}
diff --git a/packages/api/test/unit/test_runner2/runResultHelpers.spec.ts b/packages/api/test/unit/test_runner2/runResultHelpers.spec.ts
new file mode 100644
index 0000000000..9ff08cbdb6
--- /dev/null
+++ b/packages/api/test/unit/test_runner2/runResultHelpers.spec.ts
@@ -0,0 +1,56 @@
+import { expect } from 'chai';
+
+import { toMutantRunResult, RunStatus, MutantRunResult, MutantRunStatus } from '../../../test_runner2';
+import TestStatus from '../../../src/test_runner/TestStatus';
+
+describe('runResultHelpers', () => {
+  describe(toMutantRunResult.name, () => {
+    it('should convert "timeout" to "timeout"', () => {
+      const expected: MutantRunResult = { status: MutantRunStatus.Timeout };
+      expect(toMutantRunResult({ status: RunStatus.Timeout })).deep.eq(expected);
+    });
+
+    it('should convert "error" to "error"', () => {
+      const expected: MutantRunResult = { status: MutantRunStatus.Error, errorMessage: 'some error' };
+      expect(toMutantRunResult({ status: RunStatus.Error, errorMessage: 'some error' })).deep.eq(expected);
+    });
+
+    it('should report a failed test as "killed"', () => {
+      const expected: MutantRunResult = { status: MutantRunStatus.Killed, failureMessage: 'expected foo to be bar', killedBy: '42' };
+      expect(
+        toMutantRunResult({
+          status: RunStatus.Complete,
+          tests: [
+            { status: TestStatus.Success, id: 'success1', name: 'success1', timeSpentMs: 42 },
+            { status: TestStatus.Failed, id: '42', name: 'error', timeSpentMs: 42, failureMessage: 'expected foo to be bar' },
+            { status: TestStatus.Success, id: 'success2', name: 'success2', timeSpentMs: 42 },
+          ],
+        })
+      ).deep.eq(expected);
+    });
+
+    it('should report only succeeded tests as "survived"', () => {
+      const expected: MutantRunResult = { status: MutantRunStatus.Survived };
+      expect(
+        toMutantRunResult({
+          status: RunStatus.Complete,
+          tests: [
+            { status: TestStatus.Success, id: 'success1', name: 'success1', timeSpentMs: 42 },
+            { status: TestStatus.Success, id: '42', name: 'error', timeSpentMs: 42 },
+            { status: TestStatus.Success, id: 'success2', name: 'success2', timeSpentMs: 42 },
+          ],
+        })
+      ).deep.eq(expected);
+    });
+
+    it('should report an empty suite as "survived"', () => {
+      const expected: MutantRunResult = { status: MutantRunStatus.Survived };
+      expect(
+        toMutantRunResult({
+          status: RunStatus.Complete,
+          tests: [],
+        })
+      ).deep.eq(expected);
+    });
+  });
+});

From ca0b25588f990667bec0fec27eac708603a686c6 Mon Sep 17 00:00:00 2001
From: Nico Jansen <jansennico@gmail.com>
Date: Fri, 12 Jun 2020 17:08:50 +0200
Subject: [PATCH 11/11] Small improvements

---
 packages/api/schema/stryker-core.json            | 16 ++++++++++------
 packages/api/src/test_runner2/DryRunResult.ts    |  2 +-
 packages/api/src/test_runner2/RunOptions.ts      |  4 ++--
 packages/api/src/test_runner2/TestResult.ts      |  2 +-
 packages/api/src/test_runner2/TestStatus.ts      |  4 +---
 .../api/src/test_runner2/runResultHelpers.ts     |  2 +-
 packages/api/test_runner2.ts                     |  1 +
 7 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/packages/api/schema/stryker-core.json b/packages/api/schema/stryker-core.json
index 4dd56d9a1f..dcb2672ad2 100644
--- a/packages/api/schema/stryker-core.json
+++ b/packages/api/schema/stryker-core.json
@@ -26,6 +26,15 @@
         "Trace"
       ]
     },
+    "coverageAnalysis": {
+      "title": "CoverageAnalysis",
+      "type": "string",
+      "enum": [
+        "off",
+        "all",
+        "perTest"
+      ]
+    },
     "reportType": {
       "title": "ReportType",
       "type": "string",
@@ -201,13 +210,8 @@
       "default": true
     },
     "coverageAnalysis": {
+      "$ref": "#/definitions/coverageAnalysis",
       "description": "Indicates which coverage analysis strategy to use. During mutation testing, stryker will try to only run the tests that cover a particular line of code.\n\n'perTest': Analyse coverage per test.\n'all': Analyse the coverage for the entire test suite.\n'off' (default): Don't use coverage analysis",
-      "type": "string",
-      "enum": [
-        "off",
-        "all",
-        "perTest"
-      ],
       "default": "off"
     },
     "clearTextReporter": {
diff --git a/packages/api/src/test_runner2/DryRunResult.ts b/packages/api/src/test_runner2/DryRunResult.ts
index c200b0a066..3750083480 100644
--- a/packages/api/src/test_runner2/DryRunResult.ts
+++ b/packages/api/src/test_runner2/DryRunResult.ts
@@ -10,7 +10,7 @@ export interface CompleteDryRunResult {
    */
   tests: TestResult[];
 
-  mutationCoverage?: MutantCoverage;
+  mutantCoverage?: MutantCoverage;
 
   /**
    * The status of the run
diff --git a/packages/api/src/test_runner2/RunOptions.ts b/packages/api/src/test_runner2/RunOptions.ts
index ac390917af..c46b7628c0 100644
--- a/packages/api/src/test_runner2/RunOptions.ts
+++ b/packages/api/src/test_runner2/RunOptions.ts
@@ -1,4 +1,4 @@
-import { Mutant } from '../../core';
+import { Mutant, CoverageAnalysis } from '../../core';
 
 export interface RunOptions {
   /**
@@ -11,7 +11,7 @@ export interface DryRunOptions extends RunOptions {
   /**
    * Indicates whether or not mutant coverage should be collected.
    */
-  coverageAnalysis: 'off' | 'all' | 'perTest';
+  coverageAnalysis: CoverageAnalysis;
 }
 
 export interface MutantRunOptions extends RunOptions {
diff --git a/packages/api/src/test_runner2/TestResult.ts b/packages/api/src/test_runner2/TestResult.ts
index 7b4322a1ae..046fcf49bd 100644
--- a/packages/api/src/test_runner2/TestResult.ts
+++ b/packages/api/src/test_runner2/TestResult.ts
@@ -1,4 +1,4 @@
-import TestStatus from './TestStatus';
+import { TestStatus } from './TestStatus';
 
 /**
  * Indicates the result of a single test
diff --git a/packages/api/src/test_runner2/TestStatus.ts b/packages/api/src/test_runner2/TestStatus.ts
index 69bc8a823a..a25850dfe2 100644
--- a/packages/api/src/test_runner2/TestStatus.ts
+++ b/packages/api/src/test_runner2/TestStatus.ts
@@ -1,7 +1,7 @@
 /**
  * Indicates what the result of a single test was.
  */
-enum TestStatus {
+export enum TestStatus {
   /**
    * The test succeeded
    */
@@ -15,5 +15,3 @@ enum TestStatus {
    */
   Skipped,
 }
-
-export default TestStatus;
diff --git a/packages/api/src/test_runner2/runResultHelpers.ts b/packages/api/src/test_runner2/runResultHelpers.ts
index 4955186055..41481568a9 100644
--- a/packages/api/src/test_runner2/runResultHelpers.ts
+++ b/packages/api/src/test_runner2/runResultHelpers.ts
@@ -1,4 +1,4 @@
-import TestStatus from './TestStatus';
+import { TestStatus } from './TestStatus';
 import { DryRunResult } from './DryRunResult';
 import { MutantRunResult } from './MutantRunResult';
 import { RunStatus } from './RunStatus';
diff --git a/packages/api/test_runner2.ts b/packages/api/test_runner2.ts
index 4b9dd1c95a..2ce27088c7 100644
--- a/packages/api/test_runner2.ts
+++ b/packages/api/test_runner2.ts
@@ -3,6 +3,7 @@ export * from './src/test_runner2/TestRunner';
 export * from './src/test_runner2/TestStatus';
 export * from './src/test_runner2/DryRunResult';
 export * from './src/test_runner2/RunOptions';
+export * from './src/test_runner2/MutantCoverage';
 export * from './src/test_runner2/MutantRunResult';
 export * from './src/test_runner2/RunStatus';
 export * from './src/test_runner2/runResultHelpers';