feat(report): support upload of full report to dashboard (#1783)

Support having the full HTML report send to the stryker dashboard (https://dashboard.stryker-mutator.io).

See design: stryker-mutator/stryker-handbook#27 and https://github.com/stryker-mutator/stryker-handbook/blob/master/dashboard.md

Changes:

* Add `dashboard` options: `baseUrl`, `project`, `version`, `module` `reportType` (`'mutationScore'`/`'full'`). 
* Updated CI providers (travis and circle ci are supported)
* Update `StrykerCli`: made it testable and written tests for it.

packages/api/core.ts

@@ -6,3 +6,4 @@ export { default as Range } from './src/core/Range';
 export { default as MutatorDescriptor } from './src/core/MutatorDescriptor';
 export { default as MutationScoreThresholds } from './src/core/MutationScoreThresholds';
 export { default as LogLevel } from './src/core/LogLevel';
+export * from './src/core/DashboardOptions';

packages/api/package.json

@@ -33,7 +33,7 @@
     "node": ">=8"
   },
   "dependencies": {
-    "mutation-testing-report-schema": "^1.0.0",
+    "mutation-testing-report-schema": "^1.1.0",
     "tslib": "~1.10.0"
   },
   "devDependencies": {

packages/api/src/config/Config.ts

@@ -1,4 +1,14 @@
-import { LogLevel, MutationScoreThresholds, MutatorDescriptor, StrykerOptions } from '../../core';
+import { LogLevel, MutationScoreThresholds, MutatorDescriptor, StrykerOptions, DashboardOptions } from '../../core';
+import { ReportType } from '../core/DashboardOptions';
+
+/**
+ * When configuring stryker, every option is optional
+ * Including deep properties like `dashboard.project`.
+ * That's why we use a `DeepPartial` mapped type here.
+ */
+type DeepPartial<T> = {
+  [P in keyof T]?: T[P] extends object ? DeepPartial<T[P]> : T[P];
+};
 
 export default class Config implements StrykerOptions {
   [customConfig: string]: any;
@@ -29,14 +39,26 @@ export default class Config implements StrykerOptions {
     high: 80,
     low: 60
   };
+
   public allowConsoleColors: boolean = true;
+  /**
+   * The options for the 'dashboard' reporter
+   */
+  public dashboard: DashboardOptions = {
+    baseUrl: 'https://dashboard.stryker-mutator.io/api/reports',
+    reportType: ReportType.MutationScore
+  };
   public tempDirName: string = '.stryker-tmp';
 
-  public set(newConfig: Partial<StrykerOptions>) {
+  public set(newConfig: DeepPartial<StrykerOptions>) {
     if (newConfig) {
       Object.keys(newConfig).forEach(key => {
-        if (typeof newConfig[key] !== 'undefined') {
-          this[key] = newConfig[key];
+        if (newConfig[key] !== undefined) {
+          if (key === 'dashboard') {
+            this[key] = { ...this[key], ...newConfig[key] };
+          } else {
+            this[key] = newConfig[key];
+          }
         }
       });
     }

packages/api/src/core/DashboardOptions.ts

@@ -0,0 +1,32 @@
+/**
+ * The options for the dashboard reporter.
+ */
+export interface DashboardOptions {
+  /**
+   * Indicates which project to use if the "dashboard" reporter is enabled.
+   */
+  project?: string;
+  /**
+   * Indicates which version to use if the "dashboard" reporter is enabled.
+   */
+  version?: string;
+  /**
+   * Indicates which module to use if the "dashboard" reporter is enabled.
+   */
+  module?: string;
+  /**
+   * Indicates the base url of the stryker dashboard.
+   */
+  baseUrl: string;
+  /**
+   * Indicates wether to send a full report (inc. source code and mutant results) or only the mutation score.
+   */
+  reportType: ReportType;
+}
+
+export enum ReportType {
+  Full = 'full',
+  MutationScore = 'mutationScore'
+}
+
+export const ALL_REPORT_TYPES = Object.freeze([ReportType.Full, ReportType.MutationScore]);

packages/api/src/core/StrykerOptions.ts

@@ -1,6 +1,7 @@
 import LogLevel from './LogLevel';
 import MutationScoreThresholds from './MutationScoreThresholds';
 import MutatorDescriptor from './MutatorDescriptor';
+import { DashboardOptions } from './DashboardOptions';
 
 interface StrykerOptions {
   // this ensures that plugins can load custom config.
@@ -141,6 +142,11 @@ interface StrykerOptions {
    */
   allowConsoleColors: boolean;
 
+  /**
+   * The options for the 'dashboard' reporter
+   */
+  dashboard: DashboardOptions;
+
   /**
    * The name of the dir name. Default: `.stryker-tmp`
    */

packages/api/test/unit/config/Config.spec.ts

@@ -2,6 +2,7 @@ import { expect } from 'chai';
 import * as minimatch from 'minimatch';
 
 import { Config } from '../../../config';
+import { DashboardOptions, ReportType } from '../../../core';
 
 describe('Config', () => {
   let sut: Config;
@@ -32,6 +33,18 @@ describe('Config', () => {
       sut.set({ thresholds: undefined });
       expect(sut.thresholds).not.be.undefined;
     });
+
+    it('should merge `dashboard` settings', () => {
+      sut.set({
+        dashboard: { project: 'my-pet-shop' }
+      });
+      const expected: DashboardOptions = {
+        baseUrl: 'https://dashboard.stryker-mutator.io/api/reports',
+        reportType: ReportType.MutationScore,
+        project: 'my-pet-shop'
+      };
+      expect(sut.dashboard).deep.eq(expected);
+    });
   });
 
   describe('default value for `mutate` property', () => {

packages/core/.vscode/launch.json

@@ -124,9 +124,9 @@
       // "preLaunchTask": "build",
       "stopOnEntry": false,
       "args": [
-        "run",
-        "--coverageAnalysis",
-        "sdsd"
+        "--logLevel",
+        "debug",
+        "run"
       ],
       "cwd": "${workspaceRoot}/.",
       "runtimeExecutable": null,
@@ -144,4 +144,4 @@
       ]
     }
   ]
-}
+}

packages/core/README.md

@@ -89,6 +89,7 @@ You can *ignore* files by adding an exclamation mark (`!`) at the start of an ex
 ### Available Options
 * [allowConsoleColors](#allowConsoleColors)
 * [coverageAnalysis](#coverageAnalysis)
+* [dashboard.*](#dashboard)
 * [fileLogLevel](#fileLogLevel)
 * [files](#files)
 * [logLevel](#logLevel)
@@ -144,6 +145,15 @@ In addition to requiring your test runner to be able to report the code coverage
  Currently, `stryker-mocha-runner` as well as `stryker-karma-runner` support this. However, `stryker-karma-runner` support is limited to using it with `Jasmine` as the test framework
  (`Mocha` is not yet supported).
 
+<a name="dashboard"></a>
+### `dashboard` [`DashboardOptions`]
+
+Default: `{ baseUrl: 'https://dashboard.stryker-mutator.io/api/reports', reportType: 'mutationScore' }`  
+Command line: `--dashboard.project github.com/my-org/my-project --dashboard.version branch-or-tag --dashboard.module my-module --dashboard.baseUrl https://dashboard.stryker-mutator.io/api/reports --dashboard.reportType full`  
+Config file: `{ project: 'github.com/my-org/my-project', version: 'branch-or-tag', module: 'my-module', baseUrl: 'https://dashboard.stryker-mutator.io/api/reports', reportType: 'full' }`
+
+Settings for the `dashboard` [reporter](#reporters). See the [stryker handbook for more info](https://github.com/stryker-mutator/stryker-handbook/blob/master/dashboard.md)
+
 <a name="fileLogLevel"></a>
 ### `fileLogLevel` [`string`]
 
@@ -249,11 +259,12 @@ you can consult [npm](https://www.npmjs.com/search?q=%40stryker-plugin) or
 ### `reporters` [`string[]`]
 
 Default: `['clear-text', 'progress']`  
-Command line: `--reporters clear-text,progress,dots,dashboard`  
-Config file: `reporters: ['clear-text', 'progress', 'dots', 'dashboard']`     
+Command line: `--reporters clear-text,progress,dots,dashboard,html`  
+Config file: `reporters: ['clear-text', 'progress', 'dots', 'dashboard', 'html']`     
 
 With `reporters`, you can set the reporters for stryker to use.
 These reporters can be used out of the box: `clear-text`, `progress`, `dots`, `dashboard` and `event-recorder`.
+The `html` reporter can be installed as a plugin, see [the html reporter's readme](https://github.com/stryker-mutator/stryker/tree/master/packages/html-reporter#readme).
 By default, `clear-text` and `progress` are active if no reporters are configured.
 You can load additional plugins to get more reporters. See [stryker-mutator.io](https://stryker-mutator.io)
 for an up-to-date list of supported reporter plugins and a description on each reporter.
@@ -263,15 +274,7 @@ The `clear-text` reporter supports three additional config options:
 * `logTests` to log the names of unit tests that were run to allow mutants. By default, only the first three are logged. The config for your config file is: `clearTextReporter: { logTests: true },`
 * `maxTestsToLog` to show more tests that were executed to kill a mutant when `logTests` is true. The config for your config file is: `clearTextReporter: { logTests: true, maxTestsToLog: 7 },`
 
-The `dashboard` reporter is a special kind of reporter. It sends a report to https://dashboard.stryker-mutator.io, enabling you to add a fancy mutation score badge to your readme! To make sure no unwanted results are sent to the dashboards, it will only send the report if it is run from a build server. The reporter currently detects [Travis](https://travis-ci.org/) and [CircleCI](https://circleci.com/). Please open an [issue](https://github.com/stryker-mutator/stryker/issues/new) if your build server is missing. On all these environments, it will ignore builds of pull requests. Apart from build server specific environment variables, the reporter uses one environment variable:
-
-| Environment variable | Description | Example value |
-| ------------- | ------------- | ----- |
-| STRYKER\_DASHBOARD\_API\_KEY | Your API key (generated at https://dashboard.stryker-mutator.io) | `52248872-2edc-4102-a43a-bcfca7a9ca99` |
-
-You will need to pass the `STRYKER_DASHBOARD_API_KEY` environment variable yourself. You can create one for your repository by logging in on [the Stryker dashboard](https://dashboard.stryker-mutator.io). We strongly recommend you use encrypted environment variables:
-* [Travis documentation](https://docs.travis-ci.com/user/environment-variables/#Encrypting-environment-variables)
-* [CircleCI documentation](https://circleci.com/security/#secrets_section)
+The `dashboard` reporter sends a report to https://dashboard.stryker-mutator.io, enabling you to add a mutation score badge to your readme, as well as hosting your html report on the dashboard. It uses the [dashboard.*](#dashboard) configuration options. See [the Stryker handbook](https://github.com/stryker-mutator/stryker-handbook/blob/master/dashboard.md) for more info.
 
 <a name="symlinkNodeModules"></a>
 ### `symlinkNodeModules` [`boolean`]
@@ -280,7 +283,6 @@ Default: `true`
 Command line: *none*  
 Config file: `symlinkNodeModules: true`  
 
-
 The `symlinkNodeModules` value indicates whether Stryker should create a [symbolic link](https://nodejs.org/api/fs.html#fs_fs_symlink_target_path_type_callback)
 to your current node_modules directory in the sandbox directories. This makes running your tests by Stryker behave
 more like your would run the tests yourself in your project directory.
@@ -388,3 +390,4 @@ With `timeoutFactor` you can configure the allowed deviation relative to the tim
 Default: `[]`
 
 With `transpilers` you configure which transpiler plugins should transpile the code before it's executed. This is an array where the transpilers are called in the other of the array. This defaults to an empty array meaning no transpilation will be done.
+