fix(mutants): Prevent memory leak when transpiling mutants (#1376)

Instead of transpiling all mutants at once, this PR makes it so that at
most 100 mutants are transpiled at any one time.
After that, it waits for the mutants to be tested before transpiling any more of them.

Fixes #920

packages/stryker-api/plugin.ts

@@ -7,3 +7,4 @@ export * from 'typed-inject/src/api/Injector';
 export * from 'typed-inject/src/api/InjectionToken';
 export * from 'typed-inject/src/api/CorrespondingType';
 export * from 'typed-inject/src/api/Scope';
+export * from 'typed-inject/src/api/Disposable';

packages/stryker-test-helpers/src/factory.ts

@@ -155,7 +155,7 @@ export function injector(): sinon.SinonStubbedInstance<Injector> {
     provideClass: sinon.stub(),
     provideFactory: sinon.stub(),
     provideValue: sinon.stub(),
-    resolve: sinon.stub()
+    resolve: sinon.stub(),
   };
   injectorMock.provideClass.returnsThis();
   injectorMock.provideFactory.returnsThis();

packages/stryker/src/Sandbox.ts

@@ -2,7 +2,7 @@ import { StrykerOptions } from 'stryker-api/core';
 import * as path from 'path';
 import { getLogger } from 'log4js';
 import * as mkdirp from 'mkdirp';
-import { RunResult, TestRunner } from 'stryker-api/test_runner';
+import { RunResult, TestRunner, RunStatus, TestStatus } from 'stryker-api/test_runner';
 import { File } from 'stryker-api/core';
 import { TestFramework } from 'stryker-api/test_framework';
 import { wrapInClosure, normalizeWhiteSpaces } from './utils/objectUtils';
@@ -12,6 +12,7 @@ import { writeFile, findNodeModules, symlinkJunction } from './utils/fileUtils';
 import TestableMutant, { TestSelectionResult } from './TestableMutant';
 import TranspiledMutant from './TranspiledMutant';
 import LoggingClientContext from './logging/LoggingClientContext';
+import { MutantResult, MutantStatus } from 'stryker-api/report';
 
 interface FileMap {
   [sourceFile: string]: string;
@@ -22,13 +23,16 @@ export default class Sandbox {
   private readonly log = getLogger(Sandbox.name);
   private testRunner: Required<TestRunner>;
   private fileMap: FileMap;
-  private readonly files: File[];
   private readonly workingDirectory: string;
 
-  private constructor(private readonly options: StrykerOptions, private readonly index: number, files: ReadonlyArray<File>, private readonly testFramework: TestFramework | null, private readonly timeOverheadMS: number, private readonly loggingContext: LoggingClientContext) {
+  private constructor(
+    private readonly options: StrykerOptions,
+    private readonly index: number,
+    private readonly files: ReadonlyArray<File>,
+    private readonly testFramework: TestFramework | null,
+    private readonly timeOverheadMS: number, private readonly loggingContext: LoggingClientContext) {
     this.workingDirectory = TempFolder.instance().createRandomFolder('sandbox');
     this.log.debug('Creating a sandbox for files in %s', this.workingDirectory);
-    this.files = files.slice(); // Create a copy
   }
 
   private async initialize(): Promise<void> {
@@ -51,15 +55,66 @@ export default class Sandbox {
     return this.testRunner.dispose() || Promise.resolve();
   }
 
-  public async runMutant(transpiledMutant: TranspiledMutant): Promise<RunResult> {
-    const mutantFiles = transpiledMutant.transpileResult.outputFiles;
-    if (transpiledMutant.mutant.testSelectionResult === TestSelectionResult.Failed) {
-      this.log.warn(`Failed find coverage data for this mutant, running all tests. This might have an impact on performance: ${transpiledMutant.mutant.toString()}`);
+  public async runMutant(transpiledMutant: TranspiledMutant): Promise<MutantResult> {
+    const earlyResult = this.retrieveEarlyResult(transpiledMutant);
+    if (earlyResult) {
+      return earlyResult;
+    } else {
+      const mutantFiles = transpiledMutant.transpileResult.outputFiles;
+      if (transpiledMutant.mutant.testSelectionResult === TestSelectionResult.Failed) {
+        this.log.warn(`Failed find coverage data for this mutant, running all tests. This might have an impact on performance: ${transpiledMutant.mutant.toString()}`);
+      }
+      await Promise.all(mutantFiles.map(mutatedFile => this.writeFileInSandbox(mutatedFile)));
+      const runResult = await this.run(this.calculateTimeout(transpiledMutant.mutant), this.getFilterTestsHooks(transpiledMutant.mutant), this.fileMap[transpiledMutant.mutant.fileName]);
+      await this.reset(mutantFiles);
+      return this.collectMutantResult(transpiledMutant.mutant, runResult);
+    }
+  }
+
+  private readonly retrieveEarlyResult = (transpiledMutant: TranspiledMutant): MutantResult | null => {
+    if (transpiledMutant.transpileResult.error) {
+      if (this.log.isDebugEnabled()) {
+        this.log.debug(`Transpile error occurred: "${transpiledMutant.transpileResult.error}" during transpiling of mutant ${transpiledMutant.mutant.toString()}`);
+      }
+      const result = transpiledMutant.mutant.result(MutantStatus.TranspileError, []);
+      return result;
+    } else if (!transpiledMutant.mutant.selectedTests.length) {
+      const result = transpiledMutant.mutant.result(MutantStatus.NoCoverage, []);
+      return result;
+    } else if (!transpiledMutant.changedAnyTranspiledFiles) {
+      const result = transpiledMutant.mutant.result(MutantStatus.Survived, []);
+      return result;
+    } else {
+      // No early result possible, need to run in the sandbox later
+      return null;
+    }
+  }
+
+  private collectMutantResult(mutant: TestableMutant, runResult: RunResult): MutantResult {
+    const status: MutantStatus = this.determineMutantState(runResult);
+    const testNames = runResult.tests
+      .filter(t => t.status !== TestStatus.Skipped)
+      .map(t => t.name);
+    if (this.log.isDebugEnabled() && status === MutantStatus.RuntimeError) {
+      const error = runResult.errorMessages ? runResult.errorMessages.toString() : '(undefined)';
+      this.log.debug('A runtime error occurred: %s during execution of mutant: %s', error, mutant.toString());
+    }
+    return mutant.result(status, testNames);
+  }
+
+  private determineMutantState(runResult: RunResult): MutantStatus {
+    switch (runResult.status) {
+      case RunStatus.Timeout:
+        return MutantStatus.TimedOut;
+      case RunStatus.Error:
+        return MutantStatus.RuntimeError;
+      case RunStatus.Complete:
+        if (runResult.tests.some(t => t.status === TestStatus.Failed)) {
+          return MutantStatus.Killed;
+        } else {
+          return MutantStatus.Survived;
+        }
     }
-    await Promise.all(mutantFiles.map(mutatedFile => this.writeFileInSandbox(mutatedFile)));
-    const runResult = await this.run(this.calculateTimeout(transpiledMutant.mutant), this.getFilterTestsHooks(transpiledMutant.mutant), this.fileMap[transpiledMutant.mutant.fileName]);
-    await this.reset(mutantFiles);
-    return runResult;
   }
 
   private reset(mutatedFiles: ReadonlyArray<File>) {
@@ -113,7 +168,7 @@ export default class Sandbox {
 
   private async initializeTestRunner(): Promise<void> {
     const fileNames = Object.keys(this.fileMap).map(sourceFileName => this.fileMap[sourceFileName]);
-    this.log.debug(`Creating test runner %s using settings {port: %s}`, this.index);
+    this.log.debug(`Creating test runner %s`, this.index);
     this.testRunner = ResilientTestRunnerFactory.create(this.options, fileNames, this.workingDirectory, this.loggingContext);
     await this.testRunner.init();
   }

packages/stryker/src/SandboxPool.ts

@@ -1,6 +1,6 @@
 import * as os from 'os';
-import { Observable, range } from 'rxjs';
-import { flatMap } from 'rxjs/operators';
+import { range, Subject, Observable } from 'rxjs';
+import { flatMap, tap, zip, merge, map } from 'rxjs/operators';
 import { File, StrykerOptions } from 'stryker-api/core';
 import { TestFramework } from 'stryker-api/test_framework';
 import Sandbox from './Sandbox';
@@ -9,23 +9,63 @@ import { tokens, commonTokens } from 'stryker-api/plugin';
 import { coreTokens } from './di';
 import { InitialTestRunResult } from './process/InitialTestExecutor';
 import { Logger } from 'stryker-api/logging';
+import TranspiledMutant from './TranspiledMutant';
+import { MutantResult } from 'stryker-api/report';
+
+const MAX_CONCURRENT_INITIALIZING_SANDBOXES = 2;
 
 export class SandboxPool {
 
-  private readonly sandboxes: Promise<Sandbox>[] = [];
+  private readonly allSandboxes: Promise<Sandbox>[] = [];
   private readonly overheadTimeMS: number;
 
-  public static inject = tokens(commonTokens.logger, commonTokens.options, coreTokens.testFramework, coreTokens.initialRunResult, coreTokens.loggingContext);
+  public static inject = tokens(
+    commonTokens.logger,
+    commonTokens.options,
+    coreTokens.testFramework,
+    coreTokens.initialRunResult,
+    coreTokens.transpiledFiles,
+    coreTokens.loggingContext);
   constructor(
     private readonly log: Logger,
     private readonly options: StrykerOptions,
     private readonly testFramework: TestFramework | null,
     initialRunResult: InitialTestRunResult,
+    private readonly initialFiles: ReadonlyArray<File>,
     private readonly loggingContext: LoggingClientContext) {
-      this.overheadTimeMS = initialRunResult.overheadTimeMS;
-    }
+    this.overheadTimeMS = initialRunResult.overheadTimeMS;
+  }
 
-  public streamSandboxes(initialFiles: ReadonlyArray<File>): Observable<Sandbox> {
+  public runMutants(mutants: Observable<TranspiledMutant>): Observable<MutantResult> {
+    const recycledSandboxes = new Subject<Sandbox>();
+    // Make sure sandboxes get recycled
+    const sandboxes = this.startSandboxes().pipe(merge(recycledSandboxes));
+    return mutants.pipe(
+      zip(sandboxes),
+      flatMap(this.runInSandbox),
+      tap(({ sandbox }) => {
+        recycledSandboxes.next(sandbox);
+      }),
+      map(({ result }) => result)
+    );
+  }
+
+  private readonly runInSandbox = async ([mutant, sandbox]: [TranspiledMutant, Sandbox]) => {
+    const result = await sandbox.runMutant(mutant);
+    return { result, sandbox };
+  }
+
+  private startSandboxes(): Observable<Sandbox> {
+    const concurrency = this.determineConcurrency();
+
+    return range(0, concurrency).pipe(
+      flatMap(n => {
+        return this.registerSandbox(Sandbox.create(this.options, n, this.initialFiles, this.testFramework, this.overheadTimeMS, this.loggingContext));
+      }, MAX_CONCURRENT_INITIALIZING_SANDBOXES)
+    );
+  }
+
+  private determineConcurrency() {
     let numConcurrentRunners = os.cpus().length;
     if (this.options.transpilers.length) {
       // If transpilers are configured, one core is reserved for the compiler (for now)
@@ -40,18 +80,16 @@ export class SandboxPool {
       numConcurrentRunners = 1;
     }
     this.log.info(`Creating ${numConcurrentRunners} test runners (based on ${numConcurrentRunnersSource})`);
-
-    const sandboxes = range(0, numConcurrentRunners)
-      .pipe(flatMap(n => this.registerSandbox(Sandbox.create(this.options, n, initialFiles, this.testFramework, this.overheadTimeMS, this.loggingContext))));
-    return sandboxes;
+    return numConcurrentRunners;
   }
 
-  private registerSandbox(promisedSandbox: Promise<Sandbox>): Promise<Sandbox> {
-    this.sandboxes.push(promisedSandbox);
-    return promisedSandbox;
+  private readonly registerSandbox = async (promisedSandbox: Promise<Sandbox>): Promise<Sandbox> => {
+      this.allSandboxes.push(promisedSandbox);
+      return promisedSandbox;
   }
 
-  public disposeAll() {
-    return Promise.all(this.sandboxes.map(promisedSandbox => promisedSandbox.then(sandbox => sandbox.dispose())));
+  public async disposeAll() {
+    const sandboxes = await Promise.all(this.allSandboxes);
+    return Promise.all(sandboxes.map(sandbox => sandbox.dispose()));
   }
 }

packages/stryker/src/Stryker.ts

@@ -14,9 +14,10 @@ import { Injector } from 'typed-inject';
 import { TranspilerFacade } from './transpiler/TranspilerFacade';
 import { coreTokens, MainContext, PluginCreator, buildMainInjector } from './di';
 import { commonTokens, PluginKind } from 'stryker-api/plugin';
-import MutantTranspiler from './transpiler/MutantTranspiler';
+import { MutantTranspileScheduler } from './transpiler/MutantTranspileScheduler';
 import { SandboxPool } from './SandboxPool';
 import { Logger } from 'stryker-api/logging';
+import { transpilerFactory } from './transpiler';
 
 export default class Stryker {
 
@@ -65,9 +66,15 @@ export default class Stryker {
         .injectClass(InitialTestExecutor);
       const initialRunResult = await initialTestRunProcess.run();
       const mutator = inputFileInjector.injectClass(MutatorFacade);
-      const mutationTestProcessInjector = inputFileInjector
+      const transpilerProvider = inputFileInjector
         .provideValue(coreTokens.initialRunResult, initialRunResult)
-        .provideClass(coreTokens.mutantTranspiler, MutantTranspiler)
+        .provideValue(commonTokens.produceSourceMaps, false)
+        .provideFactory(coreTokens.transpiler, transpilerFactory);
+      const transpiler = transpilerProvider.resolve(coreTokens.transpiler);
+      const transpiledFiles = await transpiler.transpile(inputFiles.files);
+      const mutationTestProcessInjector = transpilerProvider
+        .provideValue(coreTokens.transpiledFiles, transpiledFiles)
+        .provideClass(coreTokens.mutantTranspileScheduler, MutantTranspileScheduler)
         .provideClass(coreTokens.sandboxPool, SandboxPool);
       const testableMutants = await mutationTestProcessInjector
         .injectClass(MutantTestMatcher)
@@ -88,20 +95,6 @@ export default class Stryker {
     return Promise.resolve([]);
   }
 
-  // private mutate(input: InputFileCollection, initialTestRunResult: InitialTestRunResult): TestableMutant[] {
-  //   const mutator = this.injector.injectClass(MutatorFacade);
-  //   const mutants = mutator.mutate(input.filesToMutate);
-  //   const mutantRunResultMatcher = new MutantTestMatcher(
-  //     mutants,
-  //     input.filesToMutate,
-  //     initialTestRunResult.runResult,
-  //     initialTestRunResult.sourceMapper,
-  //     initialTestRunResult.coverageMaps,
-  //     this.config,
-  //     this.reporter);
-  //   return mutantRunResultMatcher.matchWithMutants();
-  // }
-
   private wrapUpReporter(): Promise<void> {
     const maybePromise = this.reporter.wrapUp();
     if (isPromise(maybePromise)) {

packages/stryker/src/di/coreTokens.ts

@@ -4,12 +4,15 @@ export const configReadFromConfigFile = 'configReadFromConfigFile';
 export const configEditorApplier = 'configEditorApplier';
 export const inputFiles = 'inputFiles';
 export const initialRunResult = 'initialRunResult';
-export const mutantTranspiler = 'mutantTranspiler';
+export const transpiledFiles = 'transpiledFiles';
+export const mutantTranspileScheduler = 'mutantTranspileScheduler';
 export const sandboxPool = 'sandboxPool';
 export const testFramework = 'testFramework';
 export const timer = 'timer';
+export const timeOverheadMS = 'timeOverheadMS';
 export const loggingContext = 'loggingContext';
 export const transpiler = 'transpiler';
+export const sandboxIndex = 'sandboxIndex';
 export const reporter = 'reporter';
 export const pluginKind = 'pluginKind';
 export const pluginDescriptors = 'pluginDescriptors';