feat(karma-runner): force bail = true in all cases

Force "bail" (or "failFast") for jasmine and mocha test frameworks. This should improve performance.

packages/karma-runner/src/starters/stryker-karma.conf.ts

@@ -62,19 +62,21 @@ function setClientOptions(config: Config) {
   // Enabling clearContext (default true) will load "about:blank" in the iFrame after a test run.
   // As far as I can see clearing the context only has a visible effect (you don't see the result of the last test).
   // If this is true, disabling it is safe to do and solves the race condition issue.
-  config.set({ client: { clearContext: false } });
+  const clientOptions: Partial<ClientOptions> = { clearContext: false };
 
-  // Disable randomized tests with using jasmine.
-  // Stryker doesn't play nice with a random test order, since spec id's tent to move around
+  // Disable randomized tests with using jasmine. Stryker doesn't play nice with a random test order, since spec id's tent to move around
+  // Also set failFast, so that we're not waiting on more than 1 failed test
   if (config.frameworks?.includes('jasmine')) {
-    config.set({
-      client: {
-        jasmine: {
-          random: false,
-        },
-      } as Partial<ClientOptions>,
-    });
+    (clientOptions as any).jasmine = {
+      random: false,
+      failFast: true,
+    };
   }
+
+  if (config.frameworks?.includes('mocha')) {
+    (clientOptions as any).mocha = { bail: true };
+  }
+  config.set({ client: clientOptions });
 }
 
 function setUserKarmaConfig(config: Config) {

packages/karma-runner/test/integration/KarmaTestRunner.it.spec.ts

@@ -89,11 +89,11 @@ describe(`${KarmaTestRunner.name} integration`, () => {
       StrykerReporter.instance.removeAllListeners();
     });
     describe('dryRun', () => {
-      it('should report failed tests', async () => {
+      it('should report the first failed test (bail)', async () => {
         const runResult = await sut.dryRun(factory.dryRunOptions());
         assertions.expectCompleted(runResult);
         expectToHaveSuccessfulTests(runResult, 5);
-        expectToHaveFailedTests(runResult, ['Error: Expected 7 to be 8.', 'Error: Expected 3 to be 4.']);
+        expectToHaveFailedTests(runResult, ['Error: Expected 7 to be 8.']);
         expect(runResult.status).to.be.eq(DryRunStatus.Complete);
       });
     });

packages/karma-runner/test/integration/instrumented.it.spec.ts

@@ -119,11 +119,12 @@ describe(`${KarmaTestRunner.name} running on instrumented code`, () => {
         const result = await sut.mutantRun(factory.mutantRunOptions({ activeMutant: factory.mutant({ id: 2 }), testFilter: ['spec1'] }));
         assertions.expectKilled(result);
         result.failureMessage = result.failureMessage.split('\n')[0];
-        const expected: KilledMutantRunResult = {
+        const expected = factory.killedMutantRunResult({
           killedBy: 'spec1',
           status: MutantRunStatus.Killed,
           failureMessage: 'Error: Expected undefined to be 3.',
-        };
+          nrOfTests: 1,
+        });
         expect(result).deep.eq(expected);
       });
 
@@ -212,6 +213,12 @@ describe(`${KarmaTestRunner.name} running on instrumented code`, () => {
         expect(result.failureMessage.split('\n')[0]).eq('AssertionError: expected undefined to equal 7');
       });
 
+      it('should bail after first failing test', async () => {
+        const result = await sut.mutantRun(factory.mutantRunOptions({ activeMutant: factory.mutant({ id: 0 }) }));
+        assertions.expectKilled(result);
+        expect(result.nrOfTests).eq(1);
+      });
+
       it('should survive if the filtered tests do not kill the mutant', async () => {
         const result = await sut.mutantRun(
           factory.mutantRunOptions({
@@ -224,6 +231,7 @@ describe(`${KarmaTestRunner.name} running on instrumented code`, () => {
           })
         );
         assertions.expectSurvived(result);
+        expect(result.nrOfTests).eq(2);
       });
 
       it('should be able to kill again after a mutant survived', async () => {
@@ -237,6 +245,7 @@ describe(`${KarmaTestRunner.name} running on instrumented code`, () => {
           killedBy: 'Add should be able 1 to a number',
           status: MutantRunStatus.Killed,
           failureMessage: 'AssertionError: expected undefined to equal 3',
+          nrOfTests: 1,
         };
         expect(result).deep.eq(expected);
       });

packages/karma-runner/test/integration/read-config.it.spec.ts

@@ -0,0 +1,49 @@
+import path = require('path');
+
+import { testInjector, factory, assertions } from '@stryker-mutator/test-helpers';
+import { TestStatus } from '@stryker-mutator/api/test_runner2';
+import { expect } from 'chai';
+
+import KarmaTestRunner from '../../src/KarmaTestRunner';
+import StrykerReporter from '../../src/karma-plugins/StrykerReporter';
+
+describe('read config integration', () => {
+  afterEach(() => {
+    StrykerReporter.instance.removeAllListeners();
+  });
+  it('should not override client options in a mocha project', async () => {
+    testInjector.options.karma = {
+      configFile: path.resolve(__dirname, '..', '..', 'testResources', 'configs', 'mocha-client-options-karma.conf.js'),
+    };
+    const runner = testInjector.injector.injectClass(KarmaTestRunner);
+    await runner.init();
+    const dryRunResult = await runner.dryRun(factory.dryRunOptions());
+    assertions.expectCompleted(dryRunResult);
+    expect(dryRunResult.tests).lengthOf(2);
+    const [test1, test2] = dryRunResult.tests;
+    expect(test1.status).eq(TestStatus.Success);
+    expect(test1.id).eq('mocha client options should not override client options');
+    assertions.expectFailed(test2);
+    expect(test2.id).eq('mocha client options should override bail');
+    expect(test2.failureMessage).contains('Expected exception');
+  });
+  it('should not override client options in a jasmine project', async () => {
+    testInjector.options.karma = {
+      configFile: path.resolve(__dirname, '..', '..', 'testResources', 'configs', 'jasmine-client-options-karma.conf.js'),
+    };
+    const runner = testInjector.injector.injectClass(KarmaTestRunner);
+    await runner.init();
+    const dryRunResult = await runner.dryRun(factory.dryRunOptions());
+    assertions.expectCompleted(dryRunResult);
+    expect(dryRunResult.tests).lengthOf(3);
+    const [test1, test2, test3] = dryRunResult.tests;
+    expect([test1.name, test2.name, test3.name]).deep.eq([
+      'jasmine client options should not override client options',
+      'jasmine client options should override "random" options',
+      'jasmine client options should override "failFast" options',
+    ]);
+    expect(test1.status).eq(TestStatus.Success);
+    expect(test2.status).eq(TestStatus.Success);
+    expect(test3.status).eq(TestStatus.Success);
+  });
+});

packages/karma-runner/test/integration/sample.it.spec.ts

@@ -14,56 +14,44 @@ describe('Sample project', () => {
   });
 
   it('should be able to run karma with jasmine', async () => {
-    testInjector.options.karma = { configFile: path.resolve(__dirname, '..', '..', 'testResources', 'sampleProject', 'karma.conf.js') };
+    testInjector.options.karma = { configFile: path.resolve(__dirname, '..', '..', 'testResources', 'sampleProject', 'karma-jasmine.conf.js') };
     const runner = testInjector.injector.injectClass(KarmaTestRunner);
     await runner.init();
     const result = await runner.dryRun(factory.dryRunOptions());
     assertions.expectCompleted(result);
     const expectedTestResults: TimelessTestResult[] = [
       {
         id: 'spec0',
-        name: 'Add should be able to add two numbers and add one',
-        status: TestStatus.Failed,
-        failureMessage: 'Error: Expected 7 to be 8.',
-      },
-      {
-        id: 'spec1',
-        name: 'Add should be to add able 1 to a number and actually add 2',
-        status: TestStatus.Failed,
-        failureMessage: 'Error: Expected 3 to be 4.',
-      },
-      {
-        id: 'spec2',
         name: 'Add should be able to add two numbers',
         status: TestStatus.Success,
       },
       {
-        id: 'spec3',
+        id: 'spec1',
         name: 'Add should be able 1 to a number',
         status: TestStatus.Success,
       },
       {
-        id: 'spec4',
+        id: 'spec2',
         name: 'Add should be able negate a number',
         status: TestStatus.Success,
       },
       {
-        id: 'spec5',
+        id: 'spec3',
         name: 'Add should be able to recognize a negative number',
         status: TestStatus.Success,
       },
       {
-        id: 'spec6',
+        id: 'spec4',
         name: 'Add should be able to recognize that 0 is not a negative number',
         status: TestStatus.Success,
       },
       {
-        id: 'spec7',
+        id: 'spec5',
         name: 'Circle should have a circumference of 2PI when the radius is 1',
         status: TestStatus.Success,
       },
       {
-        id: 'spec8',
+        id: 'spec6',
         name: 'Add this test should fail',
         status: TestStatus.Failed,
         failureMessage: 'Error: Expected 7 to be 0.',

packages/karma-runner/test/unit/starters/stryker-karma.conf.spec.ts

@@ -107,10 +107,10 @@ describe('stryker-karma.conf.js', () => {
     sut(config);
 
     // Assert
-    expect(config).deep.include({ client: { clearContext: false } });
+    expect(config.client?.clearContext).false;
   });
 
-  it('should force random `false` when dealing with jasmine', () => {
+  it('should force non-random and failFast options when dealing with jasmine', () => {
     // Arrange
     sut.setGlobals({ getLogger, karmaConfigFile: 'karma.conf.js' });
     requireModuleStub.returns((conf: Config) => conf.set({ client: { jasmine: { random: true } } as ClientOptions, frameworks: ['jasmine'] }));
@@ -119,7 +119,19 @@ describe('stryker-karma.conf.js', () => {
     sut(config);
 
     // Assert
-    expect(config).deep.include({ client: { jasmine: { random: false } } });
+    expect((config.client as any).jasmine).deep.eq({ random: false, failFast: true });
+  });
+
+  it('should force bail options when dealing with mocha', () => {
+    // Arrange
+    sut.setGlobals({ getLogger, karmaConfigFile: 'karma.conf.js' });
+    requireModuleStub.returns((conf: Config) => conf.set({ client: { mocha: { bail: false } } as ClientOptions, frameworks: ['mocha'] }));
+
+    // Act
+    sut(config);
+
+    // Assert
+    expect((config.client as any).mocha).deep.include({ bail: true });
   });
 
   it('should configure the tests hooks middleware', () => {

packages/karma-runner/testResources/configs/jasmine-client-options-echo-ui.spec.js

@@ -0,0 +1,11 @@
+describe('jasmine client options', () => {
+  it('should not override client options', function() {
+    expect(jasmine.getEnv().configuration().oneFailurePerSpec).toBe(true);
+  });
+  it('should override "random" options', function() {
+    expect(jasmine.getEnv().configuration().random).toBe(false);
+  });
+  it('should override "failFast" options', function() {
+    expect(jasmine.getEnv().configuration().failFast).toBe(true);
+  });
+})

packages/karma-runner/testResources/configs/jasmine-client-options-karma.conf.js

@@ -0,0 +1,18 @@
+module.exports = function (config) {
+  config.set({
+    frameworks: ['jasmine'],
+    browsers: ['ChromeHeadless'],
+    files: [
+      require.resolve('./jasmine-client-options-echo-ui.spec.js')
+    ],
+    client: {
+      jasmine: {
+        random: true,
+        oneFailurePerSpec: true,
+        failFast: false,
+        timeoutInterval: 1000
+      }
+    },
+    singleRun: true
+  });
+};

packages/karma-runner/testResources/configs/mocha-client-options-echo-ui.spec.js

@@ -0,0 +1,10 @@
+suite('mocha client options', () => {
+  test('should not override client options', async function() {
+    expect(mocha.options.global).include('jQuery');
+  });
+  test('should override bail', async () => {
+    throw new Error('Expected exception');
+  });
+  test('should not execute this test', async () => {
+  });
+})

packages/karma-runner/testResources/configs/mocha-client-options-karma.conf.js

@@ -0,0 +1,17 @@
+module.exports = function (config) {
+  config.set({
+    frameworks: ['mocha', 'chai'],
+    browsers: ['ChromeHeadless'],
+    files: [
+      require.resolve('./mocha-client-options-echo-ui.spec.js')
+    ],
+    client: {
+      mocha: {
+        global: ['jQuery'],
+        ui: 'tdd',
+        bail: false // should be overridden by Stryker
+      }
+    },
+    singleRun: true
+  });
+};

packages/karma-runner/testResources/sampleProject/karma.conf.js → packages/karma-runner/testResources/sampleProject/karma-jasmine.conf.js

@@ -8,7 +8,8 @@ module.exports = function (config) {
     ],
     exclude: [
       __dirname + '/src/Error.js',
-      __dirname + '/src/InfiniteAdd.js'
+      __dirname + '/src/InfiniteAdd.js',
+      __dirname + '/test-jasmine/AddFailedSpec.js'
     ],
     singleRun: false,
     watch: true,