fix(mocha): support mutants with "runAllTests" (#2037)

Allow the test filter to be cleared when the next test run requires mocha to run all tests.

Fixes #2032

packages/api/src/test_framework/TestFramework.ts

@@ -18,8 +18,8 @@ interface TestFramework {
 
   /**
    * Creates a code fragment which, if included in a test run,
-   * will be responsible for filtering out tests with given ids.
-   * The first test gets id 0, the second id 1, etc.
+   * will be responsible for filtering out tests with given test selector.
+   * If te test selection array is empty it should reset the filtering for the next test run.
    *
    * @param selections A list indicating the tests to select.
    * @returns A script which, if included in the test run, will filter out the correct tests.

packages/core/src/Sandbox.ts

@@ -197,8 +197,12 @@ export default class Sandbox {
   }
 
   private getFilterTestsHooks(mutant: TestableMutant): string | undefined {
-    if (this.testFramework && !mutant.runAllTests) {
-      return wrapInClosure(this.testFramework.filter(mutant.selectedTests));
+    if (this.testFramework) {
+      if (mutant.runAllTests) {
+        return wrapInClosure(this.testFramework.filter([]));
+      } else {
+        return wrapInClosure(this.testFramework.filter(mutant.selectedTests));
+      }
     } else {
       return undefined;
     }

packages/core/src/child-proxy/ChildProcessProxy.ts

@@ -167,6 +167,10 @@ export default class ChildProcessProxy<T> implements Disposable {
     }
   }
 
+  public get stdout() {
+    return this.stdoutBuilder.toString();
+  }
+
   private reportError(error: Error) {
     this.workerTasks.filter(task => !task.isCompleted).forEach(task => task.reject(error));
   }

packages/core/test/integration/child-proxy/ChildProcessProxy.it.spec.ts

@@ -3,18 +3,15 @@ import * as path from 'path';
 import { File, LogLevel } from '@stryker-mutator/api/core';
 import { Logger } from '@stryker-mutator/api/logging';
 import { commonTokens } from '@stryker-mutator/api/plugin';
-import { testInjector } from '@stryker-mutator/test-helpers';
+import { testInjector, LoggingServer } from '@stryker-mutator/test-helpers';
 import { expect } from 'chai';
 import * as log4js from 'log4js';
 import { filter } from 'rxjs/operators';
 
-import getPort = require('get-port');
-
 import ChildProcessCrashedError from '../../../src/child-proxy/ChildProcessCrashedError';
 import ChildProcessProxy from '../../../src/child-proxy/ChildProcessProxy';
 import OutOfMemoryError from '../../../src/child-proxy/OutOfMemoryError';
 import { Task } from '../../../src/utils/Task';
-import LoggingServer from '../../helpers/LoggingServer';
 import currentLogMock from '../../helpers/logMock';
 import { Mock } from '../../helpers/producers';
 import { sleep } from '../../helpers/testUtils';
@@ -29,10 +26,10 @@ describe(ChildProcessProxy.name, () => {
   const workingDir = '..';
 
   beforeEach(async () => {
-    const port = await getPort();
+    loggingServer = new LoggingServer();
+    const port = await loggingServer.listen();
     const options = testInjector.injector.resolve(commonTokens.options);
     log = currentLogMock();
-    loggingServer = new LoggingServer(port);
     sut = ChildProcessProxy.create(require.resolve('./Echo'), { port, level: LogLevel.Debug }, options, { name: echoName }, workingDir, Echo);
   });
 

packages/core/test/integration/test-runner/ResilientTestRunnerFactory.it.spec.ts

@@ -6,12 +6,10 @@ import { strykerOptions } from '@stryker-mutator/test-helpers/src/factory';
 import { expect } from 'chai';
 import * as log4js from 'log4js';
 import { toArray } from 'rxjs/operators';
-
-import getPort = require('get-port');
+import { LoggingServer } from '@stryker-mutator/test-helpers';
 
 import LoggingClientContext from '../../../src/logging/LoggingClientContext';
 import ResilientTestRunnerFactory from '../../../src/test-runner/ResilientTestRunnerFactory';
-import LoggingServer from '../../helpers/LoggingServer';
 import { sleep } from '../../helpers/testUtils';
 
 describe('ResilientTestRunnerFactory integration', () => {
@@ -25,8 +23,8 @@ describe('ResilientTestRunnerFactory integration', () => {
 
   beforeEach(async () => {
     // Make sure there is a logging server listening
-    const port = await getPort();
-    loggingServer = new LoggingServer(port);
+    loggingServer = new LoggingServer();
+    const port = await loggingServer.listen();
     loggingContext = { port, level: LogLevel.Trace };
     options = strykerOptions({
       plugins: [require.resolve('./AdditionalTestRunners')],

packages/core/test/unit/Sandbox.spec.ts

@@ -37,12 +37,11 @@ describe(Sandbox.name, () => {
   let options: Config;
   let inputFiles: File[];
   let testRunner: Mock<TestRunnerDecorator>;
-  let testFrameworkStub: any;
+  let testFrameworkStub: sinon.SinonStubbedInstance<TestFramework>;
   let expectedFileToMutate: File;
   let notMutatedFile: File;
   let sandboxDirectory: string;
   let expectedTargetFileToMutate: string;
-  let expectedTestFrameworkHooksFile: string;
   let writeFileStub: sinon.SinonStub;
   let symlinkJunctionStub: sinon.SinonStub;
   let findNodeModulesStub: sinon.SinonStub;
@@ -55,13 +54,14 @@ describe(Sandbox.name, () => {
     options = { timeoutFactor: 23, timeoutMS: 1000, testRunner: 'sandboxUnitTestRunner', symlinkNodeModules: true } as any;
     testRunner = { init: sinon.stub(), run: sinon.stub().resolves(runResult), dispose: sinon.stub() };
     testFrameworkStub = {
-      filter: sinon.stub()
+      filter: sinon.stub(),
+      afterEach: sinon.stub(),
+      beforeEach: sinon.stub()
     };
     expectedFileToMutate = new File(path.resolve('file1'), 'original code');
     notMutatedFile = new File(path.resolve('file2'), 'to be mutated');
     sandboxDirectory = path.resolve('random-folder-3');
     expectedTargetFileToMutate = path.join(sandboxDirectory, 'file1');
-    expectedTestFrameworkHooksFile = path.join(sandboxDirectory, '___testHooksForStryker.js');
     inputFiles = [expectedFileToMutate, notMutatedFile];
     temporaryDirectoryMock = sinon.createStubInstance(TemporaryDirectory);
     temporaryDirectoryMock.createRandomDirectory.returns(sandboxDirectory);
@@ -84,12 +84,12 @@ describe(Sandbox.name, () => {
     files: readonly File[];
   }
   function createSut(overrides?: Partial<CreateArgs>) {
-    const args: CreateArgs = {
+    const { files, testFramework, overheadTimeMS }: CreateArgs = {
       files: inputFiles,
       overheadTimeMS: OVERHEAD_TIME_MS,
-      testFramework: null
+      testFramework: null,
+      ...overrides
     };
-    const { files, testFramework, overheadTimeMS } = { ...args, ...overrides };
     return Sandbox.create(options, SANDBOX_INDEX, files, testFramework, overheadTimeMS, LOGGING_CONTEXT, temporaryDirectoryMock as any);
   }
 
@@ -112,19 +112,19 @@ describe(Sandbox.name, () => {
     });
 
     it('should have created a sandbox folder', async () => {
-      await createSut(testFrameworkStub);
+      await createSut({ testFramework: testFrameworkStub });
       expect(temporaryDirectoryMock.createRandomDirectory).calledWith('sandbox');
     });
 
     it('should symlink node modules in sandbox directory if exists', async () => {
-      await createSut(testFrameworkStub);
+      await createSut({ testFramework: testFrameworkStub });
       expect(findNodeModulesStub).calledWith(process.cwd());
       expect(symlinkJunctionStub).calledWith('node_modules', path.join(sandboxDirectory, 'node_modules'));
     });
 
     it('should not symlink node modules in sandbox directory if no node_modules exist', async () => {
       findNodeModulesStub.resolves(null);
-      await createSut(testFrameworkStub);
+      await createSut({ testFramework: testFrameworkStub });
       expect(log.warn).calledWithMatch('Could not find a node_modules');
       expect(log.warn).calledWithMatch(process.cwd());
       expect(symlinkJunctionStub).not.called;
@@ -133,7 +133,7 @@ describe(Sandbox.name, () => {
     it('should log a warning if "node_modules" already exists in the working folder', async () => {
       findNodeModulesStub.resolves('node_modules');
       symlinkJunctionStub.rejects(fileAlreadyExistsError());
-      await createSut(testFrameworkStub);
+      await createSut({ testFramework: testFrameworkStub });
       expect(log.warn).calledWithMatch(
         normalizeWhitespaces(
           `Could not symlink "node_modules" in sandbox directory, it is already created in the sandbox.
@@ -147,7 +147,7 @@ describe(Sandbox.name, () => {
       findNodeModulesStub.resolves('basePath/node_modules');
       const error = new Error('unknown');
       symlinkJunctionStub.rejects(error);
-      await createSut(testFrameworkStub);
+      await createSut({ testFramework: testFrameworkStub });
       expect(log.warn).calledWithMatch(
         normalizeWhitespaces('Unexpected error while trying to symlink "basePath/node_modules" in sandbox directory.'),
         error
@@ -156,7 +156,7 @@ describe(Sandbox.name, () => {
 
     it('should symlink node modules in sandbox directory if `symlinkNodeModules` is `false`', async () => {
       options.symlinkNodeModules = false;
-      await createSut(testFrameworkStub);
+      await createSut({ testFramework: testFrameworkStub });
       expect(symlinkJunctionStub).not.called;
       expect(findNodeModulesStub).not.called;
     });
@@ -229,6 +229,7 @@ describe(Sandbox.name, () => {
       const sut = await createSut({ testFramework: testFrameworkStub });
       await sut.runMutant(transpiledMutant);
       expect(testFrameworkStub.filter).to.have.been.calledWith(transpiledMutant.mutant.selectedTests);
+      expect(testRunner.run).calledWithMatch({ testHooks: wrapInClosure(testFilterCodeFragment) });
     });
 
     it('should provide the filter code as testHooks, correct timeout and mutatedFileName', async () => {
@@ -254,32 +255,25 @@ describe(Sandbox.name, () => {
     });
 
     it('should not filter any tests when testFramework = null', async () => {
+      transpiledMutant.mutant.selectAllTests(runResult, TestSelectionResult.Success);
       const sut = await createSut();
-      const mutant = new TestableMutant('2', createMutant(), new SourceFile(new File('', '')));
-      await sut.runMutant(new TranspiledMutant(mutant, { outputFiles: [new File(expectedTargetFileToMutate, '')], error: null }, true));
-      expect(fileUtils.writeFile).not.calledWith(expectedTestFrameworkHooksFile);
+      await sut.runMutant(transpiledMutant);
+      expect(testRunner.run).calledWithMatch({ testHooks: undefined });
     });
 
-    it('should not filter any tests when runAllTests = true', async () => {
+    it('should filter no tests when runAllTests = true', async () => {
       // Arrange
       while (transpiledMutant.mutant.selectedTests.pop());
       transpiledMutant.mutant.selectAllTests(runResult, TestSelectionResult.Success);
-      const sut = await createSut();
+      testFrameworkStub.filter.returns('empty filter');
+      const sut = await createSut({ testFramework: testFrameworkStub });
 
       // Act
       await sut.runMutant(transpiledMutant);
 
       // Assert
-      expect(fileUtils.writeFile).not.calledWith(expectedTestFrameworkHooksFile);
-      expect(testRunner.run).called;
-    });
-
-    it('should not filter any tests when runAllTests = true', async () => {
-      const sut = await createSut();
-      const mutant = new TestableMutant('2', createMutant(), new SourceFile(new File('', '')));
-      mutant.selectAllTests(runResult, TestSelectionResult.Failed);
-      sut.runMutant(new TranspiledMutant(mutant, { outputFiles: [new File(expectedTargetFileToMutate, '')], error: null }, true));
-      expect(fileUtils.writeFile).not.calledWith(expectedTestFrameworkHooksFile);
+      expect(testRunner.run).calledWithMatch({ testHooks: wrapInClosure('empty filter') });
+      expect(testFrameworkStub.filter).calledWith([]);
     });
 
     it('should report a runtime error when test run errored', async () => {

packages/jasmine-framework/src/JasmineTestFramework.ts

@@ -28,10 +28,14 @@ export default class JasmineTestFramework implements TestFramework {
   }
 
   public filter(testSelections: TestSelection[]): string {
-    const names = testSelections.map(selection => selection.name);
-    return `
+    if (testSelections.length) {
+      const names = testSelections.map(selection => selection.name);
+      return `
       jasmine.getEnv().specFilter = function (spec) {
           return ${JSON.stringify(names)}.indexOf(spec.getFullName()) !== -1;
       }`;
+    } else {
+      return '';
+    }
   }
 }

packages/jasmine-framework/test/unit/JasmineTestFramework.spec.ts

@@ -21,14 +21,19 @@ describe('JasmineTestFramework', () => {
   });
 
   describe('filter()', () => {
-    it("should result in a specFilter of jasmine it's", () =>
+    it("should result in a specFilter of jasmine it's", () => {
       expect(
         sut.filter([
           { id: 5, name: 'test five' },
           { id: 8, name: 'test eight' }
         ])
       )
         .to.contain('jasmine.getEnv().specFilter = function (spec)')
-        .and.to.contain('return ["test five","test eight"].indexOf(spec.getFullName()) !== -1;'));
+        .and.to.contain('return ["test five","test eight"].indexOf(spec.getFullName()) !== -1;');
+    });
+
+    it('should result in an empty string when filter is empty', () => {
+      expect(sut.filter([])).eq('');
+    });
   });
 });

packages/jasmine-runner/test/integration/JasmineRunner.it.spec.ts

@@ -99,6 +99,17 @@ describe('JasmineRunner integration', () => {
       expectTestsFiltered(runResult.tests, 1, 3);
     });
 
+    it('should be able to clear the filter after a filtered run', async () => {
+      // Arrange
+      const testFramework = new JasmineTestFramework();
+      const filter1Test = wrapInClosure(testFramework.filter([{ id: 1, name: expectedJasmineInitResults[1].name }]));
+      const filterNoTests = wrapInClosure(testFramework.filter([]));
+
+      await sut.run({ testHooks: filter1Test });
+      const actualResult = await sut.run({ testHooks: filterNoTests });
+      expectTestResultsToEqual(actualResult.tests, expectedJasmineInitResults);
+    });
+
     it('should be able to filter tests in quick succession', async () => {
       // Arrange
       const testFramework = new JasmineTestFramework();

packages/karma-runner/package.json

@@ -5,7 +5,9 @@
   "main": "src/index.js",
   "scripts": {
     "test": "nyc --exclude-after-remap=false --check-coverage --reporter=html --report-dir=reports/coverage --lines 80 --functions 80 --branches 75 npm run mocha",
-    "mocha": "mocha \"test/helpers/**/*.js\" \"test/unit/**/*.js\" && mocha --timeout 30000 --exit \"test/helpers/**/*.js\" \"test/integration/**/*.js\"",
+    "mocha": "npm run test:unit && npm run test:integration",
+    "test:unit": "mocha \"test/helpers/**/*.js\" \"test/unit/**/*.js\"",
+    "test:integration": "mocha --timeout 30000 --exit \"test/helpers/**/*.js\" \"test/integration/**/*.js\"",
     "stryker": "node ../core/bin/stryker run"
   },
   "repository": {
@@ -37,8 +39,10 @@
     "@types/semver": "~7.1.0",
     "jasmine-core": "~3.5.0",
     "karma": "~4.1.0",
+    "karma-chai": "^0.1.0",
     "karma-jasmine": "~3.0.1",
-    "karma-phantomjs-launcher": "~1.0.4"
+    "karma-mocha": "^1.3.0",
+    "karma-phantomjs-launcher": "^1.0.4"
   },
   "peerDependencies": {
     "@stryker-mutator/core": "^2.0.0"