feat(mutants): Prevent memory leak when transpiling mutants

packages/stryker-api/package.json

@@ -34,6 +34,6 @@
 	},
 	"devDependencies": {
 		"surrial": "~0.1.1",
-		"typed-inject": "^0.2.0"
+		"typed-inject": "^0.2.1"
 	}
 }

packages/stryker-api/plugin.ts

@@ -7,3 +7,4 @@ export * from 'typed-inject/src/api/Injector';
 export * from 'typed-inject/src/api/InjectionToken';
 export * from 'typed-inject/src/api/CorrespondingType';
 export * from 'typed-inject/src/api/Scope';
+export * from 'typed-inject/src/api/Disposable';

packages/stryker-test-helpers/package.json

@@ -19,6 +19,6 @@
   "license": "ISC",
   "devDependencies": {
     "stryker-api": "^0.24.0",
-    "typed-inject": "^0.2.0"
+    "typed-inject": "^0.2.1"
   }
 }

packages/stryker-test-helpers/src/factory.ts

@@ -149,12 +149,13 @@ export function matchedMutant(numberOfTests: number, mutantId = numberOfTests.to
 
 export function injector(): sinon.SinonStubbedInstance<Injector> {
   const injectorMock: sinon.SinonStubbedInstance<Injector> = {
+    dispose: sinon.stub(),
     injectClass: sinon.stub(),
     injectFunction: sinon.stub(),
     provideClass: sinon.stub(),
     provideFactory: sinon.stub(),
     provideValue: sinon.stub(),
-    resolve: sinon.stub()
+    resolve: sinon.stub(),
   };
   injectorMock.provideClass.returnsThis();
   injectorMock.provideFactory.returnsThis();

packages/stryker/package.json

@@ -72,7 +72,7 @@
     "surrial": "^0.2.0",
     "tree-kill": "~1.2.0",
     "tslib": "~1.9.3",
-    "typed-inject": "^0.2.0",
+    "typed-inject": "^0.2.1",
     "typed-rest-client": "~1.1.2"
   },
   "devDependencies": {

packages/stryker/src/Sandbox.ts

@@ -2,7 +2,7 @@ import { StrykerOptions } from 'stryker-api/core';
 import * as path from 'path';
 import { getLogger } from 'log4js';
 import * as mkdirp from 'mkdirp';
-import { RunResult, TestRunner } from 'stryker-api/test_runner';
+import { RunResult, TestRunner, RunStatus, TestStatus } from 'stryker-api/test_runner';
 import { File } from 'stryker-api/core';
 import { TestFramework } from 'stryker-api/test_framework';
 import { wrapInClosure, normalizeWhiteSpaces } from './utils/objectUtils';
@@ -12,6 +12,7 @@ import { writeFile, findNodeModules, symlinkJunction } from './utils/fileUtils';
 import TestableMutant, { TestSelectionResult } from './TestableMutant';
 import TranspiledMutant from './TranspiledMutant';
 import LoggingClientContext from './logging/LoggingClientContext';
+import { MutantResult, MutantStatus } from 'stryker-api/report';
 
 interface FileMap {
   [sourceFile: string]: string;
@@ -22,13 +23,16 @@ export default class Sandbox {
   private readonly log = getLogger(Sandbox.name);
   private testRunner: Required<TestRunner>;
   private fileMap: FileMap;
-  private readonly files: File[];
   private readonly workingDirectory: string;
 
-  private constructor(private readonly options: StrykerOptions, private readonly index: number, files: ReadonlyArray<File>, private readonly testFramework: TestFramework | null, private readonly timeOverheadMS: number, private readonly loggingContext: LoggingClientContext) {
+  private constructor(
+    private readonly options: StrykerOptions,
+    private readonly index: number,
+    private readonly files: ReadonlyArray<File>,
+    private readonly testFramework: TestFramework | null,
+    private readonly timeOverheadMS: number, private readonly loggingContext: LoggingClientContext) {
     this.workingDirectory = TempFolder.instance().createRandomFolder('sandbox');
     this.log.debug('Creating a sandbox for files in %s', this.workingDirectory);
-    this.files = files.slice(); // Create a copy
   }
 
   private async initialize(): Promise<void> {
@@ -51,15 +55,66 @@ export default class Sandbox {
     return this.testRunner.dispose() || Promise.resolve();
   }
 
-  public async runMutant(transpiledMutant: TranspiledMutant): Promise<RunResult> {
-    const mutantFiles = transpiledMutant.transpileResult.outputFiles;
-    if (transpiledMutant.mutant.testSelectionResult === TestSelectionResult.Failed) {
-      this.log.warn(`Failed find coverage data for this mutant, running all tests. This might have an impact on performance: ${transpiledMutant.mutant.toString()}`);
+  public async runMutant(transpiledMutant: TranspiledMutant): Promise<MutantResult> {
+    const earlyResult = this.retrieveEarlyResult(transpiledMutant);
+    if (earlyResult) {
+      return earlyResult;
+    } else {
+      const mutantFiles = transpiledMutant.transpileResult.outputFiles;
+      if (transpiledMutant.mutant.testSelectionResult === TestSelectionResult.Failed) {
+        this.log.warn(`Failed find coverage data for this mutant, running all tests. This might have an impact on performance: ${transpiledMutant.mutant.toString()}`);
+      }
+      await Promise.all(mutantFiles.map(mutatedFile => this.writeFileInSandbox(mutatedFile)));
+      const runResult = await this.run(this.calculateTimeout(transpiledMutant.mutant), this.getFilterTestsHooks(transpiledMutant.mutant), this.fileMap[transpiledMutant.mutant.fileName]);
+      await this.reset(mutantFiles);
+      return this.collectMutantResult(transpiledMutant.mutant, runResult);
+    }
+  }
+
+  private readonly retrieveEarlyResult = (transpiledMutant: TranspiledMutant): MutantResult | null => {
+    if (transpiledMutant.transpileResult.error) {
+      if (this.log.isDebugEnabled()) {
+        this.log.debug(`Transpile error occurred: "${transpiledMutant.transpileResult.error}" during transpiling of mutant ${transpiledMutant.mutant.toString()}`);
+      }
+      const result = transpiledMutant.mutant.result(MutantStatus.TranspileError, []);
+      return result;
+    } else if (!transpiledMutant.mutant.selectedTests.length) {
+      const result = transpiledMutant.mutant.result(MutantStatus.NoCoverage, []);
+      return result;
+    } else if (!transpiledMutant.changedAnyTranspiledFiles) {
+      const result = transpiledMutant.mutant.result(MutantStatus.Survived, []);
+      return result;
+    } else {
+      // No early result possible, need to run in the sandbox later
+      return null;
+    }
+  }
+
+  private collectMutantResult(mutant: TestableMutant, runResult: RunResult): MutantResult {
+    const status: MutantStatus = this.determineMutantState(runResult);
+    const testNames = runResult.tests
+      .filter(t => t.status !== TestStatus.Skipped)
+      .map(t => t.name);
+    if (this.log.isDebugEnabled() && status === MutantStatus.RuntimeError) {
+      const error = runResult.errorMessages ? runResult.errorMessages.toString() : '(undefined)';
+      this.log.debug('A runtime error occurred: %s during execution of mutant: %s', error, mutant.toString());
+    }
+    return mutant.result(status, testNames);
+  }
+
+  private determineMutantState(runResult: RunResult): MutantStatus {
+    switch (runResult.status) {
+      case RunStatus.Timeout:
+        return MutantStatus.TimedOut;
+      case RunStatus.Error:
+        return MutantStatus.RuntimeError;
+      case RunStatus.Complete:
+        if (runResult.tests.some(t => t.status === TestStatus.Failed)) {
+          return MutantStatus.Killed;
+        } else {
+          return MutantStatus.Survived;
+        }
     }
-    await Promise.all(mutantFiles.map(mutatedFile => this.writeFileInSandbox(mutatedFile)));
-    const runResult = await this.run(this.calculateTimeout(transpiledMutant.mutant), this.getFilterTestsHooks(transpiledMutant.mutant), this.fileMap[transpiledMutant.mutant.fileName]);
-    await this.reset(mutantFiles);
-    return runResult;
   }
 
   private reset(mutatedFiles: ReadonlyArray<File>) {
@@ -113,7 +168,7 @@ export default class Sandbox {
 
   private async initializeTestRunner(): Promise<void> {
     const fileNames = Object.keys(this.fileMap).map(sourceFileName => this.fileMap[sourceFileName]);
-    this.log.debug(`Creating test runner %s using settings {port: %s}`, this.index);
+    this.log.debug(`Creating test runner %s`, this.index);
     this.testRunner = ResilientTestRunnerFactory.create(this.options, fileNames, this.workingDirectory, this.loggingContext);
     await this.testRunner.init();
   }

packages/stryker/src/SandboxPool.ts

@@ -1,6 +1,6 @@
 import * as os from 'os';
-import { Observable, range } from 'rxjs';
-import { flatMap } from 'rxjs/operators';
+import { range, Subject, Observable } from 'rxjs';
+import { flatMap, tap, zip, merge, map } from 'rxjs/operators';
 import { File, StrykerOptions } from 'stryker-api/core';
 import { TestFramework } from 'stryker-api/test_framework';
 import Sandbox from './Sandbox';
@@ -9,23 +9,63 @@ import { tokens, commonTokens } from 'stryker-api/plugin';
 import { coreTokens } from './di';
 import { InitialTestRunResult } from './process/InitialTestExecutor';
 import { Logger } from 'stryker-api/logging';
+import TranspiledMutant from './TranspiledMutant';
+import { MutantResult } from 'stryker-api/report';
+
+const MAX_CONCURRENT_INITIALIZING_SANDBOXES = 2;
 
 export class SandboxPool {
 
-  private readonly sandboxes: Promise<Sandbox>[] = [];
+  private readonly allSandboxes: Promise<Sandbox>[] = [];
   private readonly overheadTimeMS: number;
 
-  public static inject = tokens(commonTokens.logger, commonTokens.options, coreTokens.testFramework, coreTokens.initialRunResult, coreTokens.loggingContext);
+  public static inject = tokens(
+    commonTokens.logger,
+    commonTokens.options,
+    coreTokens.testFramework,
+    coreTokens.initialRunResult,
+    coreTokens.transpiledFiles,
+    coreTokens.loggingContext);
   constructor(
     private readonly log: Logger,
     private readonly options: StrykerOptions,
     private readonly testFramework: TestFramework | null,
     initialRunResult: InitialTestRunResult,
+    private readonly initialFiles: ReadonlyArray<File>,
     private readonly loggingContext: LoggingClientContext) {
-      this.overheadTimeMS = initialRunResult.overheadTimeMS;
-    }
+    this.overheadTimeMS = initialRunResult.overheadTimeMS;
+  }
 
-  public streamSandboxes(initialFiles: ReadonlyArray<File>): Observable<Sandbox> {
+  public runMutants(mutants: Observable<TranspiledMutant>): Observable<MutantResult> {
+    const recycledSandboxes = new Subject<Sandbox>();
+    // Make sure sandboxes get recycled
+    const sandboxes = this.startSandboxes().pipe(merge(recycledSandboxes));
+    return mutants.pipe(
+      zip(sandboxes),
+      flatMap(this.runInSandbox),
+      tap(({ sandbox }) => {
+        recycledSandboxes.next(sandbox);
+      }),
+      map(({ result }) => result)
+    );
+  }
+
+  private readonly runInSandbox = async ([mutant, sandbox]: [TranspiledMutant, Sandbox]) => {
+    const result = await sandbox.runMutant(mutant);
+    return { result, sandbox };
+  }
+
+  private startSandboxes(): Observable<Sandbox> {
+    const concurrency = this.determineConcurrency();
+
+    return range(0, concurrency).pipe(
+      flatMap(n => {
+        return this.registerSandbox(Sandbox.create(this.options, n, this.initialFiles, this.testFramework, this.overheadTimeMS, this.loggingContext));
+      }, MAX_CONCURRENT_INITIALIZING_SANDBOXES)
+    );
+  }
+
+  private determineConcurrency() {
     let numConcurrentRunners = os.cpus().length;
     if (this.options.transpilers.length) {
       // If transpilers are configured, one core is reserved for the compiler (for now)
@@ -40,18 +80,16 @@ export class SandboxPool {
       numConcurrentRunners = 1;
     }
     this.log.info(`Creating ${numConcurrentRunners} test runners (based on ${numConcurrentRunnersSource})`);
-
-    const sandboxes = range(0, numConcurrentRunners)
-      .pipe(flatMap(n => this.registerSandbox(Sandbox.create(this.options, n, initialFiles, this.testFramework, this.overheadTimeMS, this.loggingContext))));
-    return sandboxes;
+    return numConcurrentRunners;
   }
 
-  private registerSandbox(promisedSandbox: Promise<Sandbox>): Promise<Sandbox> {
-    this.sandboxes.push(promisedSandbox);
-    return promisedSandbox;
+  private readonly registerSandbox = async (promisedSandbox: Promise<Sandbox>): Promise<Sandbox> => {
+      this.allSandboxes.push(promisedSandbox);
+      return promisedSandbox;
   }
 
-  public disposeAll() {
-    return Promise.all(this.sandboxes.map(promisedSandbox => promisedSandbox.then(sandbox => sandbox.dispose())));
+  public async disposeAll() {
+    const sandboxes = await Promise.all(this.allSandboxes);
+    return Promise.all(sandboxes.map(sandbox => sandbox.dispose()));
   }
 }

packages/stryker/src/Stryker.ts

@@ -14,9 +14,10 @@ import { Injector } from 'typed-inject';
 import { TranspilerFacade } from './transpiler/TranspilerFacade';
 import { coreTokens, MainContext, PluginCreator, buildMainInjector } from './di';
 import { commonTokens, PluginKind } from 'stryker-api/plugin';
-import MutantTranspiler from './transpiler/MutantTranspiler';
+import { MutantTranspileScheduler } from './transpiler/MutantTranspileScheduler';
 import { SandboxPool } from './SandboxPool';
 import { Logger } from 'stryker-api/logging';
+import { transpilerFactory } from './transpiler';
 
 export default class Stryker {
 
@@ -65,9 +66,15 @@ export default class Stryker {
         .injectClass(InitialTestExecutor);
       const initialRunResult = await initialTestRunProcess.run();
       const mutator = inputFileInjector.injectClass(MutatorFacade);
-      const mutationTestProcessInjector = inputFileInjector
+      const transpilerProvider = inputFileInjector
         .provideValue(coreTokens.initialRunResult, initialRunResult)
-        .provideClass(coreTokens.mutantTranspiler, MutantTranspiler)
+        .provideValue(commonTokens.produceSourceMaps, false)
+        .provideFactory(coreTokens.transpiler, transpilerFactory);
+      const transpiler = transpilerProvider.resolve(coreTokens.transpiler);
+      const transpiledFiles = await transpiler.transpile(inputFiles.files);
+      const mutationTestProcessInjector = transpilerProvider
+        .provideValue(coreTokens.transpiledFiles, transpiledFiles)
+        .provideClass(coreTokens.mutantTranspileScheduler, MutantTranspileScheduler)
         .provideClass(coreTokens.sandboxPool, SandboxPool);
       const testableMutants = await mutationTestProcessInjector
         .injectClass(MutantTestMatcher)
@@ -88,20 +95,6 @@ export default class Stryker {
     return Promise.resolve([]);
   }
 
-  // private mutate(input: InputFileCollection, initialTestRunResult: InitialTestRunResult): TestableMutant[] {
-  //   const mutator = this.injector.injectClass(MutatorFacade);
-  //   const mutants = mutator.mutate(input.filesToMutate);
-  //   const mutantRunResultMatcher = new MutantTestMatcher(
-  //     mutants,
-  //     input.filesToMutate,
-  //     initialTestRunResult.runResult,
-  //     initialTestRunResult.sourceMapper,
-  //     initialTestRunResult.coverageMaps,
-  //     this.config,
-  //     this.reporter);
-  //   return mutantRunResultMatcher.matchWithMutants();
-  // }
-
   private wrapUpReporter(): Promise<void> {
     const maybePromise = this.reporter.wrapUp();
     if (isPromise(maybePromise)) {

packages/stryker/src/di/coreTokens.ts

@@ -4,12 +4,15 @@ export const configReadFromConfigFile = 'configReadFromConfigFile';
 export const configEditorApplier = 'configEditorApplier';
 export const inputFiles = 'inputFiles';
 export const initialRunResult = 'initialRunResult';
-export const mutantTranspiler = 'mutantTranspiler';
+export const transpiledFiles = 'transpiledFiles';
+export const mutantTranspileScheduler = 'mutantTranspileScheduler';
 export const sandboxPool = 'sandboxPool';
 export const testFramework = 'testFramework';
 export const timer = 'timer';
+export const timeOverheadMS = 'timeOverheadMS';
 export const loggingContext = 'loggingContext';
 export const transpiler = 'transpiler';
+export const sandboxIndex = 'sandboxIndex';
 export const reporter = 'reporter';
 export const pluginKind = 'pluginKind';
 export const pluginDescriptors = 'pluginDescriptors';