test(performance): Add performance tests (#1347)

.travis.yml

@@ -1,17 +1,33 @@
 language: node_js
+install: npm install
 node_js:
 - node
 - '6'
-install: npm install
 before_install:
 - if [[ `npm -v` = 3* ]]; then npm i -g npm@5; fi
+before_script:
+- npm ls
 sudo: 'false'
 addons:
   chrome: stable
-before_script:
-- npm ls
-after_success: ".travis/ci-release.jsh"
-env:
-  global:
-  - secure: YS5/OoRepciTMeHa9U5AV77yI2PLu+qpNDc5K+S2thRG90uDx4ePL0w6fbKs4olwE152zVieKfBrm7Q/kpcYp97IdDSryWzDyvFMa3u3/2BlA2d3zoyj06+5qH7dFHMUA8X1eix6RtcDYkKgT0++Y4Uprtl6bjSSeiaA3K97eCDXYQHrsr2ChjSgHCOrz7qpUVZER6BhVbFXbJkl+uJohs/e5wfGNfpOZTmMnWC1E3I4ev4JL0kfk1JObo0r45pitCnwgiSFGvrBpbPgcLYR/dTUJeuDKiR7oQNwWto1kxAJ8RH79wwrSNV9FN2USMnH+Eb39hyauyYz2DlYMJJ6KX6RXqUpXHFqGKfOovMb2sseepJT6EVC/Phjo5E7G7dgv1MAi8XcEztKWqKaxYSgeFYhUw5vZufdpnMhr/+bE7SaDgscKgxnnt2hlJ/KwR3gBFhspykuY7qA8WDCTYo2mswy0GTBfLOn2aC6P3cSKUrCwTuzZcaYFSIIIP26t9Sgj9JGcQ+kY/YIELBIEbOly7Ydf8jeM4DEsLClOriTbMVbA5TTao9HBSmIEK4WxN1Jh5SM67PNOhNNbLp69kUh/cZ0g68ORfCRESt69SEu8z3myumw+rpdjxF9SdzTp24Xh2HFF7XOg4/ZNmWXh7jjp1tZ7NzMaUmWt3WuBc6fZOo=
-  - secure: EhfzUfITtwsAir3v0qlWYnmCl43tbEGwM9aBwz3sDYARW8vzlsN5o4uuCkpVoE8vRuCTc8Bwg2BvCWglt6HEZx0Vspp83JuiKhRnbIAQcHQWGqpTS76ml6H62kESOQoNEMMBCmi53w3t+N2f5epVugz4GGtx9SNpoLXLNK3q32WVEZY6uPsKzksdF/F878/c7ty+9DxFNd4/ja1rTMmhkLis4IoaWZsDteazCKLzL11KXwNjUzEOXEcUFSTh+cQs4iKxuAsZWPF9y0ks6Cd9mvvUwoAG3qvsLhCzqQMw7CeavKtw+GMG2v7w3ypzhoIBaB+9rjUagyidkkfFFxe3t/Gl/VnbDp9y4gfEqxCGMQeL+lIG9Msemb3T9GT3ibwAOmkYQ0HTWGyUsXAhpceo0U3Gb1GKTmqsC1gPFI+L0H/mv5mN4SVjF0myi6wgHJm1XKKa8//86O0GPJ81SiPgxes2WZYUEf80rjzdGPVMnkclKvtyTrlxsqsYvLFJ3c8dJbk9LAdvnDpNaJpRTZ6U9dqC6YSkIGJHn0jhIO7WXv5RZjS5QApJqDsl3vFQDoUU2ZK54wFV90CjPUfk4HedYpCJ91FbV0g5Gd6kRGDRp1jGOZfYYk4/Ie7LZ27RbNIKYaJTKzBZR8sPyyanP6/iY8vVLMv6q8GOLv10OLUdImA=
+script: npm run lint && npm run build && npm run test
+jobs:
+  include:
+    - stage: test
+      name: "End 2 end tests"
+      script: npm run build && npm run e2e
+    - stage: Performance test
+      name: "npm run perf"
+      script: npm run build && npm run perf
+    - stage: release
+      script: npm run build && ".travis/ci-release.jsh"
+      env:
+        - secure: YS5/OoRepciTMeHa9U5AV77yI2PLu+qpNDc5K+S2thRG90uDx4ePL0w6fbKs4olwE152zVieKfBrm7Q/kpcYp97IdDSryWzDyvFMa3u3/2BlA2d3zoyj06+5qH7dFHMUA8X1eix6RtcDYkKgT0++Y4Uprtl6bjSSeiaA3K97eCDXYQHrsr2ChjSgHCOrz7qpUVZER6BhVbFXbJkl+uJohs/e5wfGNfpOZTmMnWC1E3I4ev4JL0kfk1JObo0r45pitCnwgiSFGvrBpbPgcLYR/dTUJeuDKiR7oQNwWto1kxAJ8RH79wwrSNV9FN2USMnH+Eb39hyauyYz2DlYMJJ6KX6RXqUpXHFqGKfOovMb2sseepJT6EVC/Phjo5E7G7dgv1MAi8XcEztKWqKaxYSgeFYhUw5vZufdpnMhr/+bE7SaDgscKgxnnt2hlJ/KwR3gBFhspykuY7qA8WDCTYo2mswy0GTBfLOn2aC6P3cSKUrCwTuzZcaYFSIIIP26t9Sgj9JGcQ+kY/YIELBIEbOly7Ydf8jeM4DEsLClOriTbMVbA5TTao9HBSmIEK4WxN1Jh5SM67PNOhNNbLp69kUh/cZ0g68ORfCRESt69SEu8z3myumw+rpdjxF9SdzTp24Xh2HFF7XOg4/ZNmWXh7jjp1tZ7NzMaUmWt3WuBc6fZOo=
+        - secure: EhfzUfITtwsAir3v0qlWYnmCl43tbEGwM9aBwz3sDYARW8vzlsN5o4uuCkpVoE8vRuCTc8Bwg2BvCWglt6HEZx0Vspp83JuiKhRnbIAQcHQWGqpTS76ml6H62kESOQoNEMMBCmi53w3t+N2f5epVugz4GGtx9SNpoLXLNK3q32WVEZY6uPsKzksdF/F878/c7ty+9DxFNd4/ja1rTMmhkLis4IoaWZsDteazCKLzL11KXwNjUzEOXEcUFSTh+cQs4iKxuAsZWPF9y0ks6Cd9mvvUwoAG3qvsLhCzqQMw7CeavKtw+GMG2v7w3ypzhoIBaB+9rjUagyidkkfFFxe3t/Gl/VnbDp9y4gfEqxCGMQeL+lIG9Msemb3T9GT3ibwAOmkYQ0HTWGyUsXAhpceo0U3Gb1GKTmqsC1gPFI+L0H/mv5mN4SVjF0myi6wgHJm1XKKa8//86O0GPJ81SiPgxes2WZYUEf80rjzdGPVMnkclKvtyTrlxsqsYvLFJ3c8dJbk9LAdvnDpNaJpRTZ6U9dqC6YSkIGJHn0jhIO7WXv5RZjS5QApJqDsl3vFQDoUU2ZK54wFV90CjPUfk4HedYpCJ91FbV0g5Gd6kRGDRp1jGOZfYYk4/Ie7LZ27RbNIKYaJTKzBZR8sPyyanP6/iY8vVLMv6q8GOLv10OLUdImA=
+# Conditional stages: https://docs.travis-ci.com/user/conditional-builds-stages-jobs/
+stages:
+  - test
+  - name: release
+    if: branch = release AND fork = false
+  - name: "Performance test"
+    if: type = cron # Only run in nightly build

CONTRIBUTING.md

@@ -28,25 +28,37 @@ We configured the tslint extension to run on save in or [vscode workspace](#vsco
 
 ## Runner Stryker locally
 
-We use [Lerna](https://lernajs.io/) to manage the packages in this repository. You don't have to install it globally. The packages themselves can be found in the [packages folder](https://github.com/stryker-mutator/stryker/tree/master/packages). npm commands such as `npm test` can be executed from the root of the project but executing them inside of a package folder is more time efficient. However, we suggest running `npm test` in the root of the project before a commit to ensure that everything still works. To get Stryker running locally, please follow these steps:
+We use [Lerna](https://lernajs.io/) to manage the packages in this repository. You don't have to install it globally. The packages themselves can be found in the [packages folder](https://github.com/stryker-mutator/stryker/tree/master/packages). Commands such as `npm test` can be executed from the root of the project but executing them inside of a package folder is more time efficient. However, we suggest running `npm test` in the root of the project before a commit to ensure that everything still works. To get Stryker running locally, please follow these steps:
 
 1. Clone the repository
 1. Install dependencies using `npm install` in the root of the project
-1. Run `npm run build` in the root of the project once. You can also run `npm test` in the root, as that will trigger a build as well.
+1. Run `npm run build` in the root of the project once.
 1. Use `npm start` to run the TypeScript compiler in watch mode. Any changes you make to TypeScript files will automatically compile.
-1. Running `npm test` in one of the package folders also works. Just remember that those will _not_ trigger a build. So you'll have 
-to run `npm start` or `npm run build` before that if you want code changes to be tested.
+
+After that, you're ready to go. If you're using [vscode, please setup that as well](#vscode-environment-configuration).
+
+Here are some common tasks to use. Just remember that they don't include compiling of the sources. So they all assume the latest code is compiled and ready to go:
+
+* Use `npm test` to run the unit tests. Using `npm test` in one of the package folders also works.
+* Use `npm run e2e` will install and execute the end to end tests (located in the e2e folder). These take a while.
+* Use `npm run perf` will install and execute the performance tests (located in the perf folder). These take a while.
 
 ## VSCode environment configuration
 
 We've chosen to **check in in our vscode configuration**. This makes development unified amongst stryker developers. VSCode is an open source code editor maintained by Microsoft. For more info and the download link, please visit https://code.visualstudio.com/.
 
+We recommend you to install the following plugins:
+
+* [editorconfig](https://marketplace.visualstudio.com/items?itemName=EditorConfig.EditorConfig), to adhere to our white spacing rules.
+* [tslint](https://marketplace.visualstudio.com/items?itemName=ms-vscode.vscode-typescript-tslint-plugin), to adhere to our tslint rules (as well as having auto fix-on-save)
+* [code spell checker](https://marketplace.visualstudio.com/items?itemName=streetsidesoftware.code-spell-checker), no particular reason, just prevent common typo's.
+
 After cloning this repo, open the workspace with `$ code workspace.code-workspace` (or open code and use file -> Open Workspace...).
 
 Some quick notes to help you get started:
 
 1. On the left side, you can see all stryker projects and plugins. Open files from there.
-1. Use `CTRL+B` (or `⌘+B` on OSX) to open up the *Run build task* menu. This runs `npm start`, compiling any changes you make in the background.
+1. Use `CTRL+B` (or `⌘+B` on OSX) to run the *build task*. This runs `npm start`, compiling any changes you make in the background.
 1. Use `CTRL+Shift+D` (or `⌘⇧D` on OSX) to open up the *debug* pane. Here you can select a config to run. For example: select "Unit tests (stryker-api)" to run the unit tests for the `stryker-api` package. 
   * You can run the tests with `CTRL+F5` (or `⌃F5` on OSX).
   * You can debug the tests with `F5` (also `F5` on OSX). Setting breakpoints in your code and inspecting variables all work as expected.

e2e/.gitignore

@@ -0,0 +1 @@
+!package-lock.json

e2e/lerna.json

@@ -0,0 +1,6 @@
+{
+  "packages": [
+    "test/*"
+  ],
+  "version": "independent"
+}