feat(typescript): Add support for TypeScript mutation testing (#376)

Add support for JavaScript transpiled languages (ESvNext, TypesScript, Flow, CoffeeScript, etc). Add the `stryker-typescript` package with mutation testing utilities for TypeScript.

New:
* Add new plugin type `Mutator`. With this plugin it is possible to mutate source code. See [Mutator.ts](https://github.com/stryker-mutator/stryker/blob/71b8803b49ce7760c13f1ae314f160d83955a472/packages/stryker-api/src/mutant/Mutator.ts) to review the API.
* Add new plugin type `Transpiler`. With this plugin it is possible to transpile source code to JavaScript. This transpiler is than be used to transpile all source code as well as each mutant. See the explanation in 
#344 for more information. See [Transpiler.ts](https://github.com/stryker-mutator/stryker/blob/71b8803b49ce7760c13f1ae314f160d83955a472/packages/stryker-api/src/transpile/Transpiler.ts) for the api details.
* Add stryker-typescript package. This package contains the plugins used for typescript mutation testing. See readme for more information about that.

BREAKING CHANGE: 
* Hoist the `Mutator` interface to a higher abstraction. With this interface it was possible to add mutators for specific ES5 AST nodes. As we're moving away from ES5, this plugin abstraction had to be hoisted to a higher level. It is no longer possible to plugin a specific ES5 node mutator.
* Update `report` interface: Rename `MutantState.Error` => `MutantState.RuntimeError`.

README.md

@@ -21,8 +21,6 @@ To install Stryker, execute the command:
 $ npm install stryker stryker-api --save-dev
 ```
 
-***Note:*** *During installation you may run into errors caused by [node-gyp](https://github.com/nodejs/node-gyp). It is safe to ignore them.*
-
 To test if Stryker is installed correctly, execute the command:
 ```sh
 $ node_modules/.bin/stryker --version
@@ -54,6 +52,7 @@ module.exports = function(config){
             '!src/fileToIgnore.js'],
     testFramework: 'mocha',
     testRunner: 'mocha',
+    mutator: 'es5',
     reporter: ['progress', 'clear-text', 'dots', 'html', 'event-recorder'],
     coverageAnalysis: 'perTest',
     plugins: ['stryker-mocha-runner', 'stryker-html-reporter']

docs/1.lifecycle.sd

@@ -0,0 +1,23 @@
+user:actor
+stryker:Stryker[a]
+plugin:PluginLoader[a]
+cfg:ConfigReader[a]
+/cfgEditor:ConfigEditor[a]
+ro:ReporterOrchestrator[a]
+/reporter:Reporter[a]
+/transpiler:Transpiler[a]
+
+user:stryker.runMutationTest(options)
+stryker:plugin.load()
+stryker:config=cfg.readConfig()
+stryker:cfgEditor.new
+stryker:cfgEditor.configure(config)
+stryker:broadCastReporter=ro.createReporter()
+ro:reporter.new
+stryker:transpiler.new
+[c:See 2.initial-test-run sequence diagram]
+stryker:runResult=stryker.initialTestRun()&
+[/c]
+[c:See 4.run-test sequence diagram]
+stryker:mutationTestResult=stryker.mutaionTest
+[/c]

docs/2.initial-test-run.sd

@@ -0,0 +1,14 @@
+stryker:Stryker[a]
+transpiler:Transpiler[a]
+sc:SandboxCoordinator[a]
+/sandbox:Sandbox[a]
+
+stryker:sourceMaps,transpiledFiles=transpiler.transpile(sourceFiles)
+stryker:runResult=sc.initialTestRun()
+sc:sandbox.new
+sc:sandbox.initialize()
+sandbox:sandbox.fillSandbox(coverageInstrumenter)
+sc:runResult=sandbox.run()
+[c See 4.RunTest sequence diagram]
+sandbox:sandbox.runTest
+[/c]

docs/3.mutation-test.sd

@@ -0,0 +1,9 @@
+stryker:Stryker[p] "Stryker"
+cfg:ConfigReader[a]
+/cfgEditor:ConfigEditor[a]
+ro:ReporterOrchestrator[a]
+/reporter:Reporter[a]
+/transpiler:Transpiler[a]
+sc:SandboxCoordinator[a]
+sandbox:Sandbox[a]
+ConfiguratorPlugin:configurator

docs/4.run-test.sd

@@ -0,0 +1,28 @@
+sandbox:Sandbox "Sandbox"
+factory:ResilientTestRunnerFactor
+/retry:RetryDecorator[a]
+/timeout:TimeoutDecorator[a]
+/isolated:IsolatedTestRunnerAdapter[a]
+/worker:IsolatedTestRunnerAdpaterWorker[a]
+plugin:PluginLoader[a]
+
+sandbox:testRunner=factory.create(config)
+factory:retry.new(config)
+retry:timeout.new(config)
+timeout:isolated.new(config)
+[c:childprocess testRunner]
+isolated:worker.new(config)
+worker:plugin.load(config)
+[/c]
+sandbox:retry.inialize()
+retry:timeout.initialize()
+timeout:isolated.initialize()
+[c:childprocess testRunner]
+isolated:worker.initialize()
+[/c]
+sandbox:retry.run()
+retry:timeout.run()
+timeout:isolated.run()
+[c:childprocess testRunner]
+isolated:worker.run()
+[/c]

integrationTest/package.json

@@ -17,8 +17,8 @@
     "load-grunt-tasks": "^3.5.2",
     "mocha": "^3.3.0",
     "mz": "^2.6.0",
-    "ts-node": "^3.0.4",
-    "typescript": "^2.2.2"
+    "ts-node": "^3.3.0",
+    "typescript": "^2.4.2"
   },
   "scripts": {
     "postinstall": "install-local && link-parent-bin -c test --link-local-dependencies true"
@@ -31,6 +31,7 @@
     "stryker-jasmine": "../packages/stryker-jasmine",
     "stryker-karma-runner": "../packages/stryker-karma-runner",
     "stryker-mocha-framework": "../packages/stryker-mocha-framework",
-    "stryker-mocha-runner": "../packages/stryker-mocha-runner"
+    "stryker-mocha-runner": "../packages/stryker-mocha-runner",
+    "stryker-typescript": "../packages/stryker-typescript"
   }
 }

integrationTest/test/typescript-transpiling/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "test-module",
+  "version": "0.0.0",
+  "private": true,
+  "description": "A module to perform an integration test",
+  "main": "index.js",
+  "scripts": {
+    "pretest": "rimraf \"reports\"",
+    "test": "stryker run stryker.conf.js",
+    "posttest": "mocha --require ts-node/register verify/*.ts"
+  },
+  "author": "",
+  "license": "ISC"
+}

integrationTest/test/typescript-transpiling/src/Add.ts

@@ -0,0 +1,25 @@
+
+export function add(num1: number, num2: number) {
+  return num1 + num2;
+}
+
+export function addOne(n: number) {
+  n++;
+  return n;
+}
+
+export function negate(n: number) {
+  return -n;
+}
+
+export function notCovered(n: number) {
+  return n > 10;
+}
+
+export function isNegativeNumber(n: number) {
+  let isNegative = false;
+  if (n < 0) {
+    isNegative = true;
+  }
+  return isNegative;
+}

integrationTest/test/typescript-transpiling/src/Circle.ts

@@ -0,0 +1,7 @@
+export function getCircumference(radius: number) {
+  return 2 * Math.PI * radius;
+}
+
+export function untestedFunction {
+  const i = 5 / 2 * 3;
+}

integrationTest/test/typescript-transpiling/stryker.conf.js

@@ -0,0 +1,17 @@
+module.exports = function (config) {
+  config.set({
+    tsconfigFile: 'tsconfig.json',
+    mutate: ['src/*.ts'],
+    testFramework: 'mocha',
+    testRunner: 'mocha',
+    coverageAnalysis: 'off',
+    reporter: ['clear-text', 'html', 'event-recorder'],
+    maxConcurrentTestRunners: 2,
+    mutator: 'typescript',
+    logLevel: 'info',
+    transpilers: [
+      'typescript'
+    ],
+    port: 9264
+  });
+};

integrationTest/test/typescript-transpiling/test/AddSpec.ts

@@ -0,0 +1,48 @@
+import { expect } from 'chai';
+import { add, addOne, isNegativeNumber, notCovered, negate } from '../src/Add';
+
+describe('Add', () => {
+  it('should be able to add two numbers', () => {
+    const num1 = 2;
+    const num2 = 5;
+    const expected = num1 + num2;
+
+    const actual = add(num1, num2);
+
+    expect(actual).to.be.equal(expected);
+  });
+
+  it('should be able 1 to a number', () => {
+    const number = 2;
+    const expected = 3;
+
+    const actual = addOne(number);
+
+    expect(actual).to.be.equal(expected);
+  });
+
+  it('should be able negate a number', () => {
+    const number = 2;
+    const expected = -2;
+
+    const actual = negate(number);
+
+    expect(actual).to.be.equal(expected);
+  });
+
+  it('should be able to recognize a negative number', () => {
+    const number = -2;
+
+    const isNegative = isNegativeNumber(number);
+
+    expect(isNegative).to.be.true;
+  });
+
+  it('should be able to recognize that 0 is not a negative number', () => {
+    const number = 0;
+
+    const isNegative = isNegativeNumber(number);
+
+    expect(isNegative).to.be.false;
+  });
+});

integrationTest/test/typescript-transpiling/test/CircleSpec.ts

@@ -0,0 +1,11 @@
+import { expect } from 'chai';
+import { getCircumference } from '../src/Circle';
+
+describe('Circle', () => {
+  it('should have a circumference of 2PI when the radius is 1', () => {
+    const radius = 1;
+    const expectedCircumference = 2 * Math.PI;
+    const circumference = getCircumference(radius);
+    expect(circumference).to.be.equal(expectedCircumference);
+  });
+});

integrationTest/test/typescript-transpiling/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "lib": [
+      "es5",
+      "es2015.promise",
+      "es2015.core"
+    ]
+  },
+  "files": [
+    "src/*.ts",
+    "test/*.ts"
+  ] 
+}

integrationTest/test/typescript-transpiling/verify/verify.ts

@@ -0,0 +1,33 @@
+import * as chai from 'chai';
+import * as fs from 'mz/fs';
+import * as chaiAsPromised from 'chai-as-promised';
+chai.use(chaiAsPromised);
+const expect = chai.expect;
+const expectFileExists = (path: string) => expect(fs.exists(path), `File ${path} does not exist`).to.eventually.eq(true);
+
+describe('Verify stryker has ran correctly', () => {
+
+  describe('html reporter', () => {
+
+    it('should report in html files', () => {
+      return Promise.all([
+        expectFileExists('reports/mutation/html/Add.ts.html'),
+        expectFileExists('reports/mutation/html/Circle.ts.html'),
+        expectFileExists('reports/mutation/html/index.html')
+      ]);
+    });
+
+    it('should copy over the resources', () => {
+      return Promise.all([
+        expectFileExists('reports/mutation/html/stryker.css'),
+        expectFileExists('reports/mutation/html/stryker.js'),
+        expectFileExists('reports/mutation/html/stryker-80x80.png'),
+        expectFileExists('reports/mutation/html/bootstrap/css/bootstrap.min.css'),
+        expectFileExists('reports/mutation/html/bootstrap/css/bootstrap.min.css'),
+        expectFileExists('reports/mutation/html/highlightjs/styles/default.css')
+      ]);
+    });
+  });
+
+
+});

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "devDependencies": {
-    "@types/chai-as-promised": "0.0.29",
+    "@types/chai-as-promised": "0.0.31",
     "@types/chalk": "^0.4.28",
     "@types/commander": "^2.9.0",
     "@types/escodegen": "0.0.6",
@@ -21,8 +21,8 @@
     "@types/sinon": "^2.2.1",
     "@types/sinon-chai": "^2.7.27",
     "browserify": "^14.3.0",
-    "chai": "^3.5.0",
-    "chai-as-promised": "^6.0.0",
+    "chai": "^4.1.1",
+    "chai-as-promised": "^7.1.1",
     "concurrently": "^3.4.0",
     "glob": "^7.1.1",
     "lerna": "^2.0.0",
@@ -33,7 +33,8 @@
     "rimraf": "^2.6.1",
     "sinon": "^2.2.0",
     "sinon-chai": "^2.10.0",
-    "tslint": "^5.0.0",
+    "source-map-support": "^0.4.16",
+    "tslint": "^5.6.0",
     "typescript": "^2.2.2"
   },
   "scripts": {

packages/stryker-api/.vscode/settings.json

@@ -1,17 +1,15 @@
 {
-  "typescript.tsdk": "node_modules/typescript/lib",
-  "files": {
-    "exclude": {
-      ".git": "",
-      "**/*.js": {
-        "when": "$(basename).ts"
-      },
-      "**/*.d.ts": {
-        "when": "$(basename).ts"
-      },
-      "**/*.map": {
-        "when": "$(basename)"
-      }
+  "typescript.tsdk": "../../node_modules/typescript/lib",
+  "files.exclude": {
+    ".git": true,
+    "**/*.js": {
+      "when": "$(basename).ts"
+    },
+    "**/*.d.ts": {
+      "when": "$(basename).ts"
+    },
+    "**/*.map": {
+      "when": "$(basename)"
     }
   }
 }

packages/stryker-api/core.ts

@@ -1,6 +1,6 @@
 export { default as StrykerOptions } from './src/core/StrykerOptions';
 export { default as Factory } from './src/core/Factory';
-export { default as InputFile } from './src/core/InputFile';
+export { BinaryFile, WebFile, TextFile, File, FileKind, FileDescriptor } from './src/core/File';
 export { default as Position } from './src/core/Position';
 export { default as Location } from './src/core/Location';
 export { default as Range } from './src/core/Range';

packages/stryker-api/mutant.ts

@@ -1,3 +1,3 @@
+export { default as Mutant } from './src/mutant/Mutant';
 export { default as Mutator } from './src/mutant/Mutator';
-export { default as MutatorFactory } from './src/mutant/MutatorFactory';
-export { IdentifiedNode, Identified } from './src/mutant/IdentifiedNode';
+export { default as MutatorFactory } from './src/mutant/MutatorFactory';

packages/stryker-api/package.json

@@ -8,7 +8,7 @@
     "prebuild": "npm run clean",
     "build": "tsc -p .",
     "postbuild": "tslint -p tsconfig.json",
-    "test": "nyc --reporter=html --report-dir=reports/coverage --check-coverage --lines 90 --functions 68 --branches 64 mocha \"test/**/*.js\""
+    "test": "nyc --reporter=html --report-dir=reports/coverage --check-coverage --lines 90 --functions 68 --branches 63 mocha \"test/**/*.js\""
   },
   "repository": {
     "type": "git",

packages/stryker-api/src/config/Config.ts

@@ -16,6 +16,8 @@ export default class Config implements StrykerOptions {
   coverageAnalysis: 'perTest' | 'all' | 'off' = 'perTest';
   testRunner: string;
   testFramework: string;
+  mutator: string = 'es5';
+  transpilers: string[] = [];
   maxConcurrentTestRunners: number = Infinity;
   thresholds: MutationScoreThresholds = {
     high: 80,