Support preexisisting inline source maps.

If tsickle is passed sources that include inline source maps (eg ts 'sources' produced by ngc), compose those source maps with the source maps produced by running tsickle.

src/source_map_utils.ts

@@ -0,0 +1,72 @@
+import {SourceMapConsumer, SourceMapGenerator} from 'source-map';
+
+const INLINE_SOURCE_MAP_REGEX =
+    new RegExp('^//# sourceMappingURL=data:application/json;base64,(.*)$', 'm');
+
+export function containsInlineSourceMap(source: string): boolean {
+  return getInlineSourceMapCount(source) > 0;
+}
+
+export function getInlineSourceMapCount(source: string): number {
+  const match = source.match(INLINE_SOURCE_MAP_REGEX);
+  if (match) {
+    // The INLINE_SOURCE_MAP_REGEX has a capture group, but we only want to count real matches
+    return match.length / 2;
+  } else {
+    return 0;
+  }
+}
+
+export function extractInlineSourceMap(source: string): string {
+  const result = INLINE_SOURCE_MAP_REGEX.exec(source)!;
+  const base64EncodedMap = result[1];
+  return Buffer.from(base64EncodedMap, 'base64').toString('utf8');
+}
+
+export function removeInlineSourceMap(source: string): string {
+  if (INLINE_SOURCE_MAP_REGEX.test(source)) {
+    return source.replace(INLINE_SOURCE_MAP_REGEX, '');
+  }
+  return source;
+}
+
+export function setInlineSourceMap(source: string, sourceMap: string): string {
+  const encodedSourceMap = Buffer.from(sourceMap, 'utf8').toString('base64');
+  if (INLINE_SOURCE_MAP_REGEX.test(source)) {
+    return source.replace(
+        INLINE_SOURCE_MAP_REGEX,
+        `//# sourceMappingURL=data:application/json;base64,${encodedSourceMap}`);
+  } else {
+    return `${source}
+//# sourceMappingURL=data:application/json;base64,${encodedSourceMap}`;
+  }
+}
+
+export function sourceMapConsumerToGenerator(sourceMapConsumer: SourceMapConsumer):
+    SourceMapGenerator {
+  return SourceMapGenerator.fromSourceMap(sourceMapConsumer);
+}
+
+/**
+ * Tsc identifies source files by their relative path to the output file.  Since
+ * there's no easy way to identify these relative paths when tsickle generates its
+ * own source maps, we patch them with the file name from the tsc source maps
+ * before composing them.
+ */
+export function sourceMapGeneratorToConsumerWithFileName(
+    sourceMapGenerator: SourceMapGenerator, fileName: string): SourceMapConsumer {
+  const rawSourceMap = sourceMapGenerator.toJSON();
+  rawSourceMap.sources = [fileName];
+  rawSourceMap.file = fileName;
+  return new SourceMapConsumer(rawSourceMap);
+}
+
+export function sourceMapTextToConsumer(sourceMapText: string): SourceMapConsumer {
+  const sourceMapJson: any = sourceMapText;
+  return new SourceMapConsumer(sourceMapJson);
+}
+
+export function sourceMapTextToGenerator(sourceMapText: string): SourceMapGenerator {
+  const sourceMapJson: any = sourceMapText;
+  return SourceMapGenerator.fromSourceMap(this.sourceMapTextToConsumer(sourceMapJson));
+}

src/tsickle_compiler_host.ts

@@ -1,12 +1,12 @@
 import * as path from 'path';
-import {SourceMapConsumer, SourceMapGenerator} from 'source-map';
+import {SourceMapGenerator} from 'source-map';
 import * as ts from 'typescript';
 
 import {convertDecorators} from './decorator-annotator';
 import {processES5} from './es5processor';
 import {ModulesManifest} from './modules_manifest';
+import * as sm from './source_map_utils';
 import {annotate} from './tsickle';
-import {extractInlineSourceMap} from './util';
 
 /**
  * Tsickle can perform 2 different precompilation transforms - decorator downleveling
@@ -96,6 +96,7 @@ export class TsickleCompilerHost implements ts.CompilerHost {
   /** externs.js files produced by tsickle, if any. */
   public externs: {[fileName: string]: string} = {};
 
+  private preexistingSourceMaps = new Map<string, SourceMapGenerator>();
   private decoratorDownlevelSourceMaps = new Map<string, SourceMapGenerator>();
   private tsickleSourceMaps = new Map<string, SourceMapGenerator>();
 
@@ -125,6 +126,7 @@ export class TsickleCompilerHost implements ts.CompilerHost {
     }
 
     const sourceFile = this.runConfiguration.oldProgram.getSourceFile(fileName);
+    this.stripAndStoreExistingSourceMap(sourceFile);
     switch (this.runConfiguration.pass) {
       case Pass.DECORATOR_DOWNLEVEL:
         return this.downlevelDecorators(
@@ -155,71 +157,70 @@ export class TsickleCompilerHost implements ts.CompilerHost {
     this.delegate.writeFile(fileName, content, writeByteOrderMark, onError, sourceFiles);
   }
 
-  sourceMapConsumerToGenerator(sourceMapConsumer: SourceMapConsumer): SourceMapGenerator {
-    return SourceMapGenerator.fromSourceMap(sourceMapConsumer);
-  }
+  getSourceMapKeyForPathAndName(outputFilePath: string, sourceFileName: string): string {
+    const fileDir = path.dirname(outputFilePath);
 
-  /**
-   * Tsc identifies source files by their relative path to the output file.  Since
-   * there's no easy way to identify these relative paths when tsickle generates its
-   * own source maps, we patch them with the file name from the tsc source maps
-   * before composing them.
-   */
-  sourceMapGeneratorToConsumerWithFileName(
-      sourceMapGenerator: SourceMapGenerator, fileName: string): SourceMapConsumer {
-    const rawSourceMap = sourceMapGenerator.toJSON();
-    rawSourceMap.sources = [fileName];
-    rawSourceMap.file = fileName;
-    return new SourceMapConsumer(rawSourceMap);
+    return this.getCanonicalFileName(path.resolve(fileDir, sourceFileName));
   }
 
-  sourceMapTextToConsumer(sourceMapText: string): SourceMapConsumer {
-    const sourceMapJson: any = sourceMapText;
-    return new SourceMapConsumer(sourceMapJson);
+  getSourceMapKeyForSourceFile(sourceFile: ts.SourceFile): string {
+    return this.getCanonicalFileName(sourceFile.path);
   }
 
-  getSourceMapKey(outputFilePath: string, sourceFileName: string): string {
-    const fileDir = path.dirname(outputFilePath);
+  stripAndStoreExistingSourceMap(sourceFile: ts.SourceFile): void {
+    if (sm.containsInlineSourceMap(sourceFile.text)) {
+      const sourceMapJson = sm.extractInlineSourceMap(sourceFile.text);
+      const sourceMap = sm.sourceMapTextToGenerator(sourceMapJson);
+      this.preexistingSourceMaps.set(this.getSourceMapKeyForSourceFile(sourceFile), sourceMap);
 
-    return this.getCanonicalFileName(path.resolve(fileDir, sourceFileName));
+      sourceFile.text = sm.removeInlineSourceMap(sourceFile.text);
+    }
   }
 
   combineSourceMaps(filePath: string, tscSourceMapText: string): string {
-    const tscSourceMapConsumer = this.sourceMapTextToConsumer(tscSourceMapText);
-    const tscSourceMapGenerator = this.sourceMapConsumerToGenerator(tscSourceMapConsumer);
+    const tscSourceMapConsumer = sm.sourceMapTextToConsumer(tscSourceMapText);
+    const tscSourceMapGenerator = sm.sourceMapConsumerToGenerator(tscSourceMapConsumer);
 
     if (this.tsickleSourceMaps.size > 0) {
       // TODO(lucassloan): remove when the .d.ts has the correct types
       for (const sourceFileName of (tscSourceMapConsumer as any).sources) {
-        const sourceMapKey = this.getSourceMapKey(filePath, sourceFileName);
+        const sourceMapKey = this.getSourceMapKeyForPathAndName(filePath, sourceFileName);
         const tsickleSourceMapGenerator = this.tsickleSourceMaps.get(sourceMapKey)!;
-        const tsickleSourceMapConsumer = this.sourceMapGeneratorToConsumerWithFileName(
-            tsickleSourceMapGenerator, sourceFileName);
+        const tsickleSourceMapConsumer =
+            sm.sourceMapGeneratorToConsumerWithFileName(tsickleSourceMapGenerator, sourceFileName);
         tscSourceMapGenerator.applySourceMap(tsickleSourceMapConsumer);
       }
     }
     if (this.decoratorDownlevelSourceMaps.size > 0) {
       // TODO(lucassloan): remove when the .d.ts has the correct types
       for (const sourceFileName of (tscSourceMapConsumer as any).sources) {
-        const sourceMapKey = this.getSourceMapKey(filePath, sourceFileName);
+        const sourceMapKey = this.getSourceMapKeyForPathAndName(filePath, sourceFileName);
         const decoratorDownlevelSourceMapGenerator =
             this.decoratorDownlevelSourceMaps.get(sourceMapKey)!;
-        const decoratorDownlevelSourceMapConsumer = this.sourceMapGeneratorToConsumerWithFileName(
+        const decoratorDownlevelSourceMapConsumer = sm.sourceMapGeneratorToConsumerWithFileName(
             decoratorDownlevelSourceMapGenerator, sourceFileName);
         tscSourceMapGenerator.applySourceMap(decoratorDownlevelSourceMapConsumer);
       }
     }
+    if (this.preexistingSourceMaps.size > 0) {
+      // TODO(lucassloan): remove when the .d.ts has the correct types
+      for (const sourceFileName of (tscSourceMapConsumer as any).sources) {
+        const sourceMapKey = this.getSourceMapKeyForPathAndName(filePath, sourceFileName);
+        const preexistingSourceMapGenerator = this.preexistingSourceMaps.get(sourceMapKey)!;
+        const preexistingSourceMapConsumer = sm.sourceMapGeneratorToConsumerWithFileName(
+            preexistingSourceMapGenerator, sourceFileName);
+        tscSourceMapGenerator.applySourceMap(preexistingSourceMapConsumer);
+      }
+    }
+
 
     return tscSourceMapGenerator.toString();
   }
 
   combineInlineSourceMaps(filePath: string, compiledJsWithInlineSourceMap: string): string {
-    const sourceMapJson = extractInlineSourceMap(compiledJsWithInlineSourceMap);
+    const sourceMapJson = sm.extractInlineSourceMap(compiledJsWithInlineSourceMap);
     const composedSourceMap = this.combineSourceMaps(filePath, sourceMapJson);
-    const encodedSourceMap = Buffer.from(composedSourceMap, 'utf8').toString('base64');
-    return compiledJsWithInlineSourceMap.replace(
-        new RegExp('^//# sourceMappingURL=data:application/json;base64,(.*)$', 'm'),
-        `//# sourceMappingURL=data:application/json;base64,${encodedSourceMap}`);
+    return sm.setInlineSourceMap(compiledJsWithInlineSourceMap, composedSourceMap);
   }
 
   convertCommonJsToGoogModule(fileName: string, content: string): string {
@@ -242,7 +243,7 @@ export class TsickleCompilerHost implements ts.CompilerHost {
       sourceFile: ts.SourceFile, program: ts.Program, fileName: string,
       languageVersion: ts.ScriptTarget): ts.SourceFile {
     this.decoratorDownlevelSourceMaps.set(
-        this.getCanonicalFileName(sourceFile.path), new SourceMapGenerator());
+        this.getSourceMapKeyForSourceFile(sourceFile), new SourceMapGenerator());
     if (this.environment.shouldSkipTsickleProcessing(fileName)) return sourceFile;
     let fileContent = sourceFile.text;
     const converted = convertDecorators(program.getTypeChecker(), sourceFile);
@@ -255,15 +256,15 @@ export class TsickleCompilerHost implements ts.CompilerHost {
     }
     fileContent = converted.output + ANNOTATION_SUPPORT;
     this.decoratorDownlevelSourceMaps.set(
-        this.getCanonicalFileName(sourceFile.path), converted.sourceMap);
+        this.getSourceMapKeyForSourceFile(sourceFile), converted.sourceMap);
     return ts.createSourceFile(fileName, fileContent, languageVersion, true);
   }
 
   private closurize(
       sourceFile: ts.SourceFile, program: ts.Program, fileName: string,
       languageVersion: ts.ScriptTarget): ts.SourceFile {
     this.tsickleSourceMaps.set(
-        this.getCanonicalFileName(sourceFile.path), new SourceMapGenerator());
+        this.getSourceMapKeyForSourceFile(sourceFile), new SourceMapGenerator());
     let isDefinitions = /\.d\.ts$/.test(fileName);
     // Don't tsickle-process any d.ts that isn't a compilation target;
     // this means we don't process e.g. lib.d.ts.
@@ -282,7 +283,7 @@ export class TsickleCompilerHost implements ts.CompilerHost {
       diagnostics = diagnostics.filter(d => d.category === ts.DiagnosticCategory.Error);
     }
     this.diagnostics = diagnostics;
-    this.tsickleSourceMaps.set(this.getCanonicalFileName(sourceFile.path), sourceMap);
+    this.tsickleSourceMaps.set(this.getSourceMapKeyForSourceFile(sourceFile), sourceMap);
     return ts.createSourceFile(fileName, output, languageVersion, true);
   }
 

src/util.ts

@@ -88,10 +88,3 @@ export function createOutputRetainingCompilerHost(
     outputFiles.set(fileName, content);
   }
 }
-
-export function extractInlineSourceMap(source: string): string {
-  const regex = new RegExp('^//# sourceMappingURL=data:application/json;base64,(.*)$', 'm');
-  const result = regex.exec(source)!;
-  const base64EncodedMap = result[1];
-  return Buffer.from(base64EncodedMap, 'base64').toString('utf8');
-}