feat(🥒): add support for cucumber-js test runner (#2970)

Add support for the [cucumber-js](https://github.com/cucumber/cucumber-js) test runner. See https://stryker-mutator.io/docs/stryker-js/cucumber-runner

docs/cucumber-runner.md

@@ -0,0 +1,109 @@
+---
+title: Cucumber Runner
+custom_edit_url: https://github.com/stryker-mutator/stryker-js/edit/master/docs/cucumber-runner.md
+---
+
+A plugin to use CucumberJS in StrykerJS
+
+🥒 💖 👽
+
+_Note: this plugin only supports the [`@cucumber/cucumber` nodejs test runner](https://github.com/cucumber/cucumber-js/blob/main/docs/cli.md#cli). If you're running cucumber with Jest or Karma, be sure to use those respective test runners instead._
+
+## Install
+
+Install @stryker-mutator/cucumber-runner locally within your project folder, like so:
+
+```shell
+npm i --save-dev @stryker-mutator/cucumber-runner
+```
+
+## Peer dependencies
+
+The `@stryker-mutator/cucumber-runner` is a plugin for `stryker` to enable `@cucumber/cucumber` as a test runner.
+As such, you should make sure you have the correct versions of its dependencies installed:
+
+* `@cucumber/cucumber`
+* `@stryker-mutator/core`
+
+## Configuring
+
+You can configure the cucumber test runner in the `stryker.conf.json` (or `stryker.conf.js`) file.
+
+```json
+{
+  "testRunner": "cucumber",
+  "cucumber": {
+    "profile": "stryker",
+    "features": ["my-features/my-feature.feature"],
+    "tags": ["my-tag"]
+  }
+}
+```
+
+The cucumber runner supports loading [cucumber profiles](https://github.com/cucumber/cucumber-js/blob/main/docs/profiles.md#profiles) from your `cucumber.js` configuration file. The `default` profile will automatically be loaded if none was specified.
+
+### `cucumber.profile` [`string`]
+
+Default: `"default"`
+
+Choose which profile to run. See [cucumber profiles](https://github.com/cucumber/cucumber-js/blob/main/docs/profiles.md#profiles)
+
+### `cucumber.features` [`string[]`]
+
+Default: `["features"]`
+
+Choose which features to load. See [Running specific features](https://github.com/cucumber/cucumber-js/blob/main/docs/cli.md#running-specific-features).
+
+### `cucumber.tags` [`string[]`]
+
+Default: `undefined`
+
+Choose which tags to focus. See [Tags](https://github.com/cucumber/cucumber-js/blob/main/docs/cli.md#tags).
+
+
+## Coverage analysis
+
+The `@stryker-mutator/cucumber-runner` plugin supports coverage analysis, test filtering and test location reporting. This means that `--coverageAnalysis perTest` (which is the default) is supported and will yield the best performance.
+
+## Non-standard feature file locations
+
+Your feature files might be located in a directory other than `features`. For example: `my-features`. In that case, you might have your default profile in cucumber.js config file defined as:
+
+```js
+module.exports = {
+  default: 'my-features/**/*.feature'
+}
+```
+
+And use this StrykerJS configuration
+
+```json
+{
+  "testRunner": "cucumber"
+}
+```
+
+This will work. Stryker will use your default profile and load any options you provide there. Unfortunately [this will remove this plugins ability to filter tests](https://github.com/cucumber/cucumber-js/issues/1712), meaning that pretty much all the tests are executed for each mutant.
+
+A solution is to specify a _shadow profile_ and use that for Stryker:
+
+```diff
+module.exports = {
+  default: 'my-features/**/*.feature',
++ stryker: ''
+}
+```
+
+And use this StrykerJS configuration
+
+```diff
+{
+  "testRunner": "cucumber",
++  "cucumber": {
++   "profile": "stryker",
++   "features": ["my-features/**/*.feature"]
++  }
+}
+```
+
+This will make sure your feature files are run during the dry run and still allow the plugin to filter specific tests during the mutation testing phase.

docs/sidebar.json

@@ -5,6 +5,7 @@
   "stryker-js/config-file",
   "stryker-js/configuration",
   "stryker-js/plugins",
+  "stryker-js/cucumber-runner",
   "stryker-js/jasmine-runner",
   "stryker-js/jest-runner",
   "stryker-js/karma-runner",

e2e/helpers.ts

@@ -54,6 +54,13 @@ export async function readMutationTestResult(eventResultDirectory = path.resolve
   return metricsResult;
 }
 
+async function readMutationTestingJsonResult(jsonReportFile = path.resolve('reports', 'mutation', 'mutation.json')) {
+  const mutationTestReportContent = await fsPromises.readFile(jsonReportFile, 'utf8');
+  const report = JSON.parse(mutationTestReportContent) as mutationTestReportSchema.MutationTestResult;
+  const metricsResult = calculateMetrics(report.files);
+  return metricsResult;
+}
+
 type WritableMetricsResult = {
   -readonly [K in keyof MetricsResult]: MetricsResult[K];
 };
@@ -79,8 +86,20 @@ export async function expectMetricsResult(expectedMetricsResult: Partial<Metrics
   expect(actualSnippet).deep.eq(expectedMetricsResult);
 }
 
+export async function expectMetricsJson(expectedMetrics: Partial<Metrics>) {
+  const actualMetricsResult = await readMutationTestingJsonResult();
+  expectActualMetrics(expectedMetrics, actualMetricsResult);
+}
+
+/**
+ * @deprecated please use expectMetricsJson instead (and activate the json reporter)
+ */
 export async function expectMetrics(expectedMetrics: Partial<Metrics>) {
   const actualMetricsResult = await readMutationTestResult();
+  expectActualMetrics(expectedMetrics, actualMetricsResult);
+}
+
+function expectActualMetrics(expectedMetrics: Partial<Metrics>, actualMetricsResult: MetricsResult) {
   const actualMetrics: Partial<Metrics> = {};
   Object.entries(expectedMetrics).forEach(([key]) => {
     if (key === 'mutationScore' || key === 'mutationScoreBasedOnCoveredCode') {