deduplicate file lists in tsbuildinfo

microsoft · Mar 5, 2021 · e4c7402 · e4c7402
1 parent f0a72e2
commit e4c7402
Show file tree

Hide file tree

Showing 130 changed files with 3,808 additions and 2,049 deletions.
diff --git a/src/compiler/builder.ts b/src/compiler/builder.ts
@@ -695,12 +695,95 @@ namespace ts {
     export interface ProgramBuildInfo {
         fileInfos: MapLike<BuilderState.FileInfo>;
         options: CompilerOptions;
-        referencedMap?: MapLike<string[]>;
-        exportedModulesMap?: MapLike<string[]>;
+        referencedMap?: MapLike<number>;
+        exportedModulesMap?: MapLike<number>;
+        mapLists: number[][];
+        mapFiles: string[];
         semanticDiagnosticsPerFile?: ProgramBuildInfoDiagnostic[];
         affectedFilesPendingEmit?: ProgramBuilderInfoFilePendingEmit[];
     }
 
+    /**
+     * Runs deduplication on a map of lists of files to reduce storage cost
+     */
+    function deduplicateFileMaps(inputMaps: (ReadonlyESMap<Path, BuilderState.ReferencedSet> | undefined)[], relativeToBuildInfo: (path: Path) => string): { maps: (MapLike<number> | undefined)[], lists: number[][], files: string[] } {
+
+        // Discover all the files and Sets to bring them in a deterministic order
+        const allAbsoluteFiles = new Set<Path>();
+        const allSets = new Set<BuilderState.ReferencedSet>();
+        for(const inputMap of inputMaps) {
+            if(!inputMap) continue;
+            for(const key of arrayFrom(inputMap.keys())) {
+                allAbsoluteFiles.add(key);
+                const set = inputMap.get(key)!;
+                if(!allSets.has(set)) {
+                    allSets.add(set);
+                    const it = set.keys();
+                    let itResult;
+                    while (!(itResult = it.next()).done) {
+                        allAbsoluteFiles.add(itResult.value);
+                    }
+                }
+            }
+        }
+
+        // Get files list and map indicies
+        const fileToIndexMap = new Map<string, number>();
+        const filesWithInfo = arrayFrom(allAbsoluteFiles.keys(), file => ({ file, relativeFile: relativeToBuildInfo(file) })).sort((a, b) => compareStringsCaseSensitive(a.relativeFile, b.relativeFile));
+        for(let i = 0; i < filesWithInfo.length; i++) fileToIndexMap.set(filesWithInfo[i].file, i);
+        const files = filesWithInfo.map(info => info.relativeFile);
+
+        const setToArray = new Map<BuilderState.ReferencedSet, number[]>();
+        const setKeyToArray = new Map<string, number[]>();
+        const it = allSets.keys();
+        let itResult;
+        while (!(itResult = it.next()).done) {
+            const set = itResult.value;
+            const array = arrayFrom(set.keys(), file => fileToIndexMap.get(file)!).sort((a, b) => a - b);
+            const key = array.join();
+            let listArray = setKeyToArray.get(key);
+            if(listArray === undefined) {
+                listArray = array;
+                setKeyToArray.set(key, listArray);
+            }
+            setToArray.set(set, listArray);
+        }
+
+        // Create lists in a deterministic order
+        const lists = arrayFrom(setKeyToArray.values()).sort((a, b) => {
+            const d = a.length - b.length;
+            if(d !== 0) return d;
+            let i = 0;
+            while(true) {
+                const d = a[i] - b[i];
+                if(d !== 0) return d;
+                i++;
+            }
+        });
+        const arrayToIndexMap = new Map<number[], number>();
+        for(let i = 0; i < lists.length; i++) arrayToIndexMap.set(lists[i], i);
+
+        // Replace input maps with numbers
+        const maps = inputMaps.map(inputMap => {
+            if(!inputMap) return undefined;
+
+            const map: MapLike<number> = {};
+            const it = inputMap.keys();
+            let itResult;
+            while (!(itResult = it.next()).done) {
+                const key = itResult.value;
+                const set = inputMap.get(key)!;
+
+                // In an object, keys that look like numbers will always be in numeric order
+                // no need to sort them
+                map[fileToIndexMap.get(key)!] = arrayToIndexMap.get(setToArray.get(set)!)!;
+            }
+            return map;
+        });
+
+        return { maps, lists, files };
+    }
+
     /**
      * Gets the program information to be emitted in buildInfo so that we can use it to create new program
      */
@@ -714,29 +797,15 @@ namespace ts {
             fileInfos[relativeToBuildInfo(key)] = signature === undefined ? value : { version: value.version, signature, affectsGlobalScope: value.affectsGlobalScope };
         });
 
+        const { maps: [referencedMap, exportedModulesMap], lists, files } = deduplicateFileMaps([state.referencedMap, state.exportedModulesMap], relativeToBuildInfo);
         const result: ProgramBuildInfo = {
             fileInfos,
-            options: convertToReusableCompilerOptions(state.compilerOptions, relativeToBuildInfoEnsuringAbsolutePath)
+            options: convertToReusableCompilerOptions(state.compilerOptions, relativeToBuildInfoEnsuringAbsolutePath),
+            referencedMap,
+            exportedModulesMap,
+            mapLists: lists,
+            mapFiles: files,
         };
-        if (state.referencedMap) {
-            const referencedMap: MapLike<string[]> = {};
-            for (const key of arrayFrom(state.referencedMap.keys()).sort(compareStringsCaseSensitive)) {
-                referencedMap[relativeToBuildInfo(key)] = arrayFrom(state.referencedMap.get(key)!.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
-            }
-            result.referencedMap = referencedMap;
-        }
-
-        if (state.exportedModulesMap) {
-            const exportedModulesMap: MapLike<string[]> = {};
-            for (const key of arrayFrom(state.exportedModulesMap.keys()).sort(compareStringsCaseSensitive)) {
-                const newValue = state.currentAffectedFilesExportedModulesMap && state.currentAffectedFilesExportedModulesMap.get(key);
-                // Not in temporary cache, use existing value
-                if (newValue === undefined) exportedModulesMap[relativeToBuildInfo(key)] = arrayFrom(state.exportedModulesMap.get(key)!.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
-                // Value in cache and has updated value map, use that
-                else if (newValue) exportedModulesMap[relativeToBuildInfo(key)] = arrayFrom(newValue.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
-            }
-            result.exportedModulesMap = exportedModulesMap;
-        }
 
         if (state.semanticDiagnosticsPerFile) {
             const semanticDiagnosticsPerFile: ProgramBuildInfoDiagnostic[] = [];
@@ -1167,17 +1236,19 @@ namespace ts {
         }
     }
 
-    function getMapOfReferencedSet(mapLike: MapLike<readonly string[]> | undefined, toPath: (path: string) => Path): ReadonlyESMap<Path, BuilderState.ReferencedSet> | undefined {
-        if (!mapLike) return undefined;
-        const map = new Map<Path, BuilderState.ReferencedSet>();
-        // Copies keys/values from template. Note that for..in will not throw if
-        // template is undefined, and instead will just exit the loop.
-        for (const key in mapLike) {
-            if (hasProperty(mapLike, key)) {
-                map.set(toPath(key), new Set(mapLike[key].map(toPath)));
+    function expandFileMaps(inputMaps: (MapLike<number> | undefined)[], lists: number[][], files: string[], toPath: (path: string) => Path): (ReadonlyESMap<Path, BuilderState.ReferencedSet> | undefined)[] {
+        const paths = files.map(toPath);
+        const sets = lists.map(list => new Set(list.map(i => paths[i])));
+        return inputMaps.map(inputMap => {
+            if(!inputMap) return undefined;
+            const map = new Map<Path, BuilderState.ReferencedSet>();
+            for (const key in inputMap) {
+                if (hasProperty(inputMap, key)) {
+                    map.set(paths[+key], sets[+inputMap[key]]);
+                }
             }
-        }
-        return map;
+            return map;
+        });
     }
 
     export function createBuildProgramUsingProgramBuildInfo(program: ProgramBuildInfo, buildInfoPath: string, host: ReadBuildProgramHost): EmitAndSemanticDiagnosticsBuilderProgram {
@@ -1191,11 +1262,12 @@ namespace ts {
             }
         }
 
+        const [referencedMap, exportedModulesMap] = expandFileMaps([program.referencedMap, program.exportedModulesMap], program.mapLists, program.mapFiles, toPath);
         const state: ReusableBuilderProgramState = {
             fileInfos,
             compilerOptions: convertToOptionsWithAbsolutePaths(program.options, toAbsolutePath),
-            referencedMap: getMapOfReferencedSet(program.referencedMap, toPath),
-            exportedModulesMap: getMapOfReferencedSet(program.exportedModulesMap, toPath),
+            referencedMap,
+            exportedModulesMap,
             semanticDiagnosticsPerFile: program.semanticDiagnosticsPerFile && arrayToMap(program.semanticDiagnosticsPerFile, value => toPath(isString(value) ? value : value[0]), value => isString(value) ? emptyArray : value[1]),
             hasReusableDiagnostic: true,
             affectedFilesPendingEmit: map(program.affectedFilesPendingEmit, value => toPath(value[0])),

diff --git a/...onfigFileErrors/incremental-declaration-changes/builds-after-fixing-config-file-errors.js b/...onfigFileErrors/incremental-declaration-changes/builds-after-fixing-config-file-errors.js
@@ -60,6 +60,8 @@ exports.bar = bar;
     },
     "referencedMap": {},
     "exportedModulesMap": {},
+    "mapLists": [],
+    "mapFiles": [],
     "semanticDiagnosticsPerFile": [
       "../lib/lib.d.ts",
       "./a.ts",

diff --git a/...initial-build/when-building-project-uses-reference-and-both-extend-config-with-include.js b/...initial-build/when-building-project-uses-reference-and-both-extend-config-with-include.js
@@ -100,6 +100,8 @@ exports.a = 1;
     },
     "referencedMap": {},
     "exportedModulesMap": {},
+    "mapLists": [],
+    "mapFiles": [],
     "semanticDiagnosticsPerFile": [
       "../../../lib/lib.d.ts",
       "../../shared/index.ts",
@@ -149,6 +151,8 @@ exports.b = 1;
     },
     "referencedMap": {},
     "exportedModulesMap": {},
+    "mapLists": [],
+    "mapFiles": [],
     "semanticDiagnosticsPerFile": [
       "../../../lib/lib.d.ts",
       "../../shared/typings-base/globals.d.ts",

diff --git a/...Extends/initial-build/when-building-solution-with-projects-extends-config-with-include.js b/...Extends/initial-build/when-building-solution-with-projects-extends-config-with-include.js
@@ -101,6 +101,8 @@ exports.a = 1;
     },
     "referencedMap": {},
     "exportedModulesMap": {},
+    "mapLists": [],
+    "mapFiles": [],
     "semanticDiagnosticsPerFile": [
       "../../../lib/lib.d.ts",
       "../../shared/index.ts",
@@ -150,6 +152,8 @@ exports.b = 1;
     },
     "referencedMap": {},
     "exportedModulesMap": {},
+    "mapLists": [],
+    "mapFiles": [],
     "semanticDiagnosticsPerFile": [
       "../../../lib/lib.d.ts",
       "../../shared/typings-base/globals.d.ts",

diff --git a/.../initial-build/verify-that-subsequent-builds-after-initial-build-doesnt-build-anything.js b/.../initial-build/verify-that-subsequent-builds-after-initial-build-doesnt-build-anything.js
@@ -132,6 +132,8 @@ exports.x = 10;
     },
     "referencedMap": {},
     "exportedModulesMap": {},
+    "mapLists": [],
+    "mapFiles": [],
     "semanticDiagnosticsPerFile": [
       "../../../lib/lib.d.ts",
       "./index.ts"
@@ -172,6 +174,8 @@ exports.x = 10;
     },
     "referencedMap": {},
     "exportedModulesMap": {},
+    "mapLists": [],
+    "mapFiles": [],
     "semanticDiagnosticsPerFile": [
       "../../../lib/lib.d.ts",
       "./index.ts"
@@ -212,6 +216,8 @@ exports.x = 10;
     },
     "referencedMap": {},
     "exportedModulesMap": {},
+    "mapLists": [],
+    "mapFiles": [],
     "semanticDiagnosticsPerFile": [
       "../../lib/lib.d.ts",
       "./index.ts"

diff --git a/...-build/when-declaration-file-is-referenced-through-triple-slash-but-uses-no-references.js b/...-build/when-declaration-file-is-referenced-through-triple-slash-but-uses-no-references.js
@@ -148,24 +148,31 @@ exports.getVar = getVar;
       "configFilePath": "../tsconfig.json"
     },
     "referencedMap": {
-      "../src/common/nominal.ts": [
-        "../src/common/types.d.ts"
-      ],
-      "../src/subproject/index.ts": [
-        "../src/common/nominal.ts"
-      ],
-      "../src/subproject2/index.ts": [
-        "../src/subproject/index.ts"
-      ]
+      "0": 1,
+      "2": 0,
+      "3": 2
     },
     "exportedModulesMap": {
-      "../src/subproject/index.ts": [
-        "../src/common/nominal.ts"
+      "2": 0,
+      "3": 2
+    },
+    "mapLists": [
+      [
+        0
+      ],
+      [
+        1
       ],
-      "../src/subproject2/index.ts": [
-        "../src/subproject/index.ts"
+      [
+        2
       ]
-    },
+    ],
+    "mapFiles": [
+      "../src/common/nominal.ts",
+      "../src/common/types.d.ts",
+      "../src/subproject/index.ts",
+      "../src/subproject2/index.ts"
+    ],
     "semanticDiagnosticsPerFile": [
       "../../../lib/lib.d.ts",
       "../src/common/nominal.ts",