feat(jasmine-runner): implement new test runner api

packages/jasmine-runner/package.json

@@ -40,7 +40,6 @@
     "jasmine": ">=2"
   },
   "devDependencies": {
-    "@stryker-mutator/jasmine-framework": "^3.3.0",
     "@stryker-mutator/test-helpers": "^3.3.0",
     "@types/node": "^14.0.1"
   },

packages/jasmine-runner/src/JasmineTestRunner.ts

@@ -1,15 +1,26 @@
 import { EOL } from 'os';
 
-import { StrykerOptions } from '@stryker-mutator/api/core';
+import { StrykerOptions, CoverageAnalysis } from '@stryker-mutator/api/core';
 import { commonTokens, tokens } from '@stryker-mutator/api/plugin';
-import { RunResult, RunStatus, TestResult, TestRunner } from '@stryker-mutator/api/test_runner';
-import { errorToString } from '@stryker-mutator/util';
+import {
+  RunStatus,
+  TestResult,
+  TestRunner2,
+  MutantRunOptions,
+  DryRunResult,
+  MutantRunResult,
+  toMutantRunResult,
+  ErrorDryRunResult,
+  DryRunOptions,
+  MutantCoverage,
+} from '@stryker-mutator/api/test_runner2';
+import { errorToString, Task } from '@stryker-mutator/util';
 
 import { JasmineRunnerOptions } from '../src-generated/jasmine-runner-options';
 
-import { evalGlobal, Jasmine, toStrykerTestResult } from './helpers';
+import { Jasmine, toStrykerTestResult } from './helpers';
 
-export default class JasmineTestRunner implements TestRunner {
+export default class JasmineTestRunner implements TestRunner2 {
   private readonly jasmineConfigFile: string | undefined;
   private readonly Date: typeof Date = Date; // take Date prototype now we still can (user might choose to mock it away)
 
@@ -18,49 +29,72 @@ export default class JasmineTestRunner implements TestRunner {
     this.jasmineConfigFile = (options as JasmineRunnerOptions).jasmineConfigFile;
   }
 
-  public run(options: { testHooks?: string }): Promise<RunResult> {
+  public dryRun(options: DryRunOptions): Promise<DryRunResult> {
+    return this.run(undefined, options.coverageAnalysis);
+  }
+
+  public async mutantRun(options: MutantRunOptions): Promise<MutantRunResult> {
+    global.__activeMutant__ = options.activeMutant.id;
+    const runResult = await this.run(options.testFilter);
+    return toMutantRunResult(runResult);
+  }
+
+  private async run(testFilter?: string[], coverageAnalysis?: CoverageAnalysis): Promise<DryRunResult> {
     this.clearRequireCache();
-    const tests: TestResult[] = [];
-    let startTimeCurrentSpec = 0;
-    const jasmine = this.createJasmineRunner();
-    const self = this;
-    if (options.testHooks) {
-      evalGlobal(options.testHooks);
-    }
-    return new Promise<RunResult>((resolve) => {
+    try {
+      const jasmine = this.createJasmineRunner(testFilter);
+      const self = this;
+      const tests: TestResult[] = [];
+      const runTask = new Task<DryRunResult>();
+      let startTimeCurrentSpec = 0;
       const reporter: jasmine.CustomReporter = {
-        specStarted() {
+        specStarted(spec) {
+          if (coverageAnalysis && coverageAnalysis === 'perTest') {
+            global.__currentTestId__ = spec.id;
+          }
           startTimeCurrentSpec = new self.Date().getTime();
         },
-
         specDone(result: jasmine.CustomReporterResult) {
           tests.push(toStrykerTestResult(result, new self.Date().getTime() - startTimeCurrentSpec));
         },
-
         jasmineDone() {
-          resolve({
-            errorMessages: [],
+          let mutantCoverage: MutantCoverage | undefined = undefined;
+          if (coverageAnalysis === 'all' || coverageAnalysis === 'perTest') {
+            mutantCoverage = global.__mutantCoverage__;
+          }
+          runTask.resolve({
             status: RunStatus.Complete,
             tests,
+            mutantCoverage,
           });
         },
       };
+
       jasmine.env.addReporter(reporter);
       jasmine.execute();
-    }).catch((error) => ({
-      errorMessages: [`An error occurred while loading your jasmine specs${EOL}${errorToString(error)}`],
-      status: RunStatus.Error,
-      tests: [],
-    }));
+      const result = await runTask.promise;
+      return result;
+    } catch (error) {
+      const errorResult: ErrorDryRunResult = {
+        errorMessage: `An error occurred while loading your jasmine specs${EOL}${errorToString(error)}`,
+        status: RunStatus.Error,
+      };
+      return errorResult;
+    }
   }
 
-  private createJasmineRunner() {
+  private createJasmineRunner(testFilter: undefined | string[]) {
+    let specFilter: undefined | Function = undefined;
+    if (testFilter) {
+      specFilter = (spec: jasmine.Spec) => testFilter.includes(spec.id.toString());
+    }
     const jasmine = new Jasmine({ projectBaseDir: process.cwd() });
     // The `loadConfigFile` will fallback on the default
     jasmine.loadConfigFile(this.jasmineConfigFile);
     jasmine.env.configure({
       failFast: true,
       oneFailurePerSpec: true,
+      specFilter,
     });
 
     jasmine.exit = () => {};
@@ -69,7 +103,7 @@ export default class JasmineTestRunner implements TestRunner {
     return jasmine;
   }
 
-  public clearRequireCache() {
+  private clearRequireCache() {
     this.fileNames.forEach((fileName) => {
       delete require.cache[fileName];
     });

packages/jasmine-runner/src/helpers.ts

@@ -1,28 +1,31 @@
-import { TestResult, TestStatus } from '@stryker-mutator/api/test_runner';
+import { TestResult } from '@stryker-mutator/api/test_runner2';
+import { TestStatus } from '@stryker-mutator/api/test_runner';
 
 import JasmineConstructor = require('jasmine');
 
 export function toStrykerTestResult(specResult: jasmine.CustomReporterResult, timeSpentMs: number): TestResult {
-  let status = TestStatus.Failed;
-  let failureMessages: string[] | undefined;
+  const baseResult = {
+    id: specResult.id.toString(),
+    name: specResult.fullName,
+    timeSpentMs,
+  };
   if (specResult.status === 'disabled' || specResult.status === 'pending' || specResult.status === 'excluded') {
-    status = TestStatus.Skipped;
+    return {
+      ...baseResult,
+      status: TestStatus.Skipped,
+    };
   } else if (!specResult.failedExpectations || specResult.failedExpectations.length === 0) {
-    status = TestStatus.Success;
+    return {
+      ...baseResult,
+      status: TestStatus.Success,
+    };
   } else {
-    failureMessages = specResult.failedExpectations.map((failedExpectation) => failedExpectation.message);
+    return {
+      ...baseResult,
+      status: TestStatus.Failed,
+      failureMessage: specResult.failedExpectations.map((failedExpectation) => failedExpectation.message).join(','),
+    };
   }
-  return {
-    failureMessages,
-    name: specResult.fullName,
-    status,
-    timeSpentMs,
-  };
-}
-
-export function evalGlobal(body: string) {
-  const fn = new Function('require', body);
-  fn(require);
 }
 
 export const Jasmine = JasmineConstructor;

packages/jasmine-runner/src/index.ts

@@ -4,6 +4,6 @@ import * as strykerValidationSchema from '../schema/jasmine-runner-options.json'
 
 import JasmineTestRunner from './JasmineTestRunner';
 
-export const strykerPlugins = [declareClassPlugin(PluginKind.TestRunner, 'jasmine', JasmineTestRunner)];
+export const strykerPlugins = [declareClassPlugin(PluginKind.TestRunner2, 'jasmine', JasmineTestRunner)];
 
 export { strykerValidationSchema };

packages/jasmine-runner/test/helpers/assertions.ts

@@ -1,20 +1,19 @@
-import { TestResult, TestStatus } from '@stryker-mutator/api/test_runner';
+import { TestResult, SuccessTestResult, FailedTestResult, SkippedTestResult } from '@stryker-mutator/api/test_runner2';
 import { expect } from 'chai';
 
-export function expectTestResultsToEqual(
-  actualTestResults: TestResult[],
-  expectedResults: ReadonlyArray<{ name: string; status: TestStatus; failureMessages: string[] | undefined }>
-) {
+type TimelessTestResult = Omit<SuccessTestResult, 'timeSpentMs'> | Omit<FailedTestResult, 'timeSpentMs'> | Omit<SkippedTestResult, 'timeSpentMs'>;
+
+export function expectTestResultsToEqual(actualTestResults: TestResult[], expectedResults: readonly TimelessTestResult[]) {
   expect(actualTestResults).lengthOf(
     expectedResults.length,
     `Expected ${JSON.stringify(actualTestResults, null, 2)} to equal ${JSON.stringify(expectedResults, null, 2)}`
   );
   expectedResults.forEach((expectedResult) => {
     const actualTestResult = actualTestResults.find((testResult) => testResult.name === expectedResult.name);
     if (actualTestResult) {
-      expect({ name: actualTestResult.name, status: actualTestResult.status, failureMessages: actualTestResult.failureMessages }).deep.equal(
-        expectedResult
-      );
+      const actualWithoutTiming = { ...actualTestResult };
+      delete actualWithoutTiming.timeSpentMs;
+      expect(actualWithoutTiming).deep.equal(expectedResult);
     } else {
       expect.fail(
         undefined,

packages/jasmine-runner/test/helpers/global.ts

@@ -0,0 +1,5 @@
+declare namespace NodeJS {
+  interface Global {
+    __testsInCurrentJasmineRun: string[];
+  }
+}

packages/jasmine-runner/test/helpers/mockFactories.ts

@@ -11,6 +11,15 @@ export function createRunDetails(): jasmine.RunDetails {
   };
 }
 
+export function createCustomReporterResult(overrides?: Partial<jasmine.CustomReporterResult>): jasmine.CustomReporterResult {
+  return {
+    description: 'should have bar',
+    fullName: 'Foo should have bar',
+    id: 'spec0',
+    ...overrides,
+  };
+}
+
 export function createEnvStub(): sinon.SinonStubbedInstance<jasmine.Env> {
   return {
     currentSpec: sinon.stub(),

packages/jasmine-runner/test/helpers/setupTests.ts

@@ -9,6 +9,11 @@ chai.use(sinonChai);
 chai.use(chaiAsPromised);
 
 afterEach(() => {
-  sinon.reset();
+  delete global.__activeMutant__;
+  delete global.__currentTestId__;
+  delete global.__mutantCoverage__;
+  delete global.__testsInCurrentJasmineRun;
+
+  sinon.restore();
   testInjector.reset();
 });