feat(core): add --dryRunOnly CLI argument to only run initial tests (…

…#3814)

Dry run only will still mutate your code before doing the dry run without those mutants being active, thus can be used to test that StrykerJS can run your test setup. This can be useful, for example, in CI pipelines

Co-authored-by: Nico Jansen <jansennico@gmail.com>

docs/configuration.md

@@ -145,6 +145,14 @@ Config file: `"disableTypeChecks": false`
 
 Set to 'true' to disable type checking, or 'false' to enable it. For more control, configure a pattern that matches the files of which type checking has to be disabled. This is needed because Stryker will create (typescript) type errors when inserting the mutants in your code. Stryker disables type checking by inserting `// @ts-nocheck` atop those files and removing other `// @ts-xxx` directives (so they won't interfere with `@ts-nocheck`). The default setting allows these directives to be stripped from all JavaScript and friend files in `lib`, `src` and `test` directories.
 
+### `dryRunOnly` [`boolean`]
+
+Default: `false`<br />
+Command line: `--dryRunOnly`<br />
+Config file: `"dryRunOnly": false`
+
+Execute the initial test run only without doing actual mutation testing. Dry run only will still mutate your code before doing the dry run without those mutants being active, thus can be used to test that StrykerJS can run your test setup. This can be useful, for example, in CI pipelines.
+
 ### `dryRunTimeoutMinutes` [`number`]
 
 Default: `5`<br />

packages/api/schema/stryker-core.json

@@ -290,6 +290,11 @@
       "$ref": "#/definitions/dashboardOptions",
       "default": {}
     },
+    "dryRunOnly": {
+      "description": "Execute the initial test run only without doing actual mutation testing. Dry run only will still mutate your code before doing the dry run without those mutants being active, thus can be used to test that StrykerJS can run your test setup. This can be useful, for example, in CI pipelines.",
+      "type":"boolean",
+      "default": false
+    },
     "eventReporter": {
       "description": "The options for the event recorder reporter.",
       "$ref": "#/definitions/eventRecorderOptions",

packages/core/src/process/3-dry-run-executor.ts

@@ -69,8 +69,8 @@ export class DryRunExecutor {
 
   public async execute(): Promise<Injector<MutationTestContext>> {
     const testRunnerInjector = this.injector
-      .provideFactory(coreTokens.testRunnerFactory, createTestRunnerFactory)
       .provideValue(coreTokens.testRunnerConcurrencyTokens, this.concurrencyTokenProvider.testRunnerToken$)
+      .provideFactory(coreTokens.testRunnerFactory, createTestRunnerFactory)
       .provideFactory(coreTokens.testRunnerPool, createTestRunnerPool);
     const testRunnerPool = testRunnerInjector.resolve(coreTokens.testRunnerPool);
     const { result, timing } = await lastValueFrom(testRunnerPool.schedule(of(0), (testRunner) => this.executeDryRun(testRunner)));
@@ -110,6 +110,10 @@ export class DryRunExecutor {
   }
 
   private async executeDryRun(testRunner: TestRunner): Promise<DryRunCompletedEvent> {
+    if (this.options.dryRunOnly) {
+      this.log.info('Note: running the dry-run only. No mutations will be tested.');
+    }
+
     const dryRunTimeout = this.options.dryRunTimeoutMinutes * 1000 * 60;
     const project = this.injector.resolve(coreTokens.project);
     const dryRunFiles = objectUtils.map(project.filesToMutate, (_, name) => this.sandbox.sandboxFileFor(name));

packages/core/src/process/4-mutation-test-executor.ts

@@ -64,6 +64,11 @@ export class MutationTestExecutor {
   ) {}
 
   public async execute(): Promise<MutantResult[]> {
+    if (this.options.dryRunOnly) {
+      this.log.info('The dry-run has been completed successfully. No mutations have been executed.');
+      return [];
+    }
+
     const mutantTestPlans = await this.planner.makePlan(this.mutants);
     const { earlyResult$, runMutant$ } = this.executeEarlyResult(from(mutantTestPlans));
     const { passedMutant$, checkResult$ } = this.executeCheck(runMutant$);

packages/core/src/stryker-cli.ts

@@ -92,6 +92,10 @@ export class StrykerCli {
         'Configure a build command to run after mutating the code, but before mutants are tested. This is generally used to transpile your code before testing.' +
           " Only configure this if your test runner doesn't take care of this already and you're not using just-in-time transpiler like `babel/register` or `ts-node`."
       )
+      .option(
+        '--dryRunOnly',
+        'Execute the initial test run only, without doing actual mutation testing. Doing a dry run only can be used to test that StrykerJS can run your test setup, for example, in CI pipelines.'
+      )
       .option(
         '--checkers <listOfCheckersOrEmptyString>',
         'A comma separated list of checkers to use, for example --checkers typescript',

packages/core/test/unit/config/options-validator.spec.ts

@@ -54,6 +54,7 @@ describe(OptionsValidator.name, () => {
           reportType: ReportType.Full,
         },
         disableTypeChecks: '{test,src,lib}/**/*.{js,ts,jsx,tsx,html,vue}',
+        dryRunOnly: false,
         dryRunTimeoutMinutes: 5,
         eventReporter: {
           baseDir: 'reports/mutation/events',

packages/core/test/unit/process/4-mutation-test-executor.spec.ts

@@ -383,4 +383,22 @@ describe(MutationTestExecutor.name, () => {
     // Assert
     expect(testInjector.logger.info).calledWithExactly('Done in %s.', '2 seconds, tops!');
   });
+
+  it('should short circuit when dryRunOnly is enabled', async () => {
+    // Arrange
+    testInjector.options.dryRunOnly = true;
+    arrangeScenario();
+    const plan1 = mutantRunPlan({ id: '1' });
+    const plan2 = mutantRunPlan({ id: '2' });
+    mutantTestPlans.push(plan1, plan2);
+
+    // Act
+    const actualResults = await sut.execute();
+
+    // Assert
+    expect(mutantTestPlannerMock.makePlan).not.called;
+    expect(testRunner.mutantRun).not.called;
+    expect(checker.check).not.called;
+    expect(actualResults).empty;
+  });
 });

packages/core/test/unit/stryker-cli.spec.ts

@@ -36,6 +36,7 @@ describe(StrykerCli.name, () => {
       [['--checkers', ''], { checkers: [] }],
       [['--checkerNodeArgs', '--inspect=1337 --gc'], { checkerNodeArgs: ['--inspect=1337', '--gc'] }],
       [['--disableBail'], { disableBail: true }],
+      [['--dryRunOnly'], { dryRunOnly: true }],
       [['--mutate', 'foo.js,bar.js'], { mutate: ['foo.js', 'bar.js'] }],
       [['--reporters', 'foo,bar'], { reporters: ['foo', 'bar'] }],
       [['--plugins', 'foo,bar'], { plugins: ['foo', 'bar'] }],