Skip to content

Commit

Permalink
Optimize the size for tsbuildinfo (#43155)
Browse files Browse the repository at this point in the history 
* Baseline readable buildinfo

* Use file names as index in file name list
This is extension of the idea given by @sokra to optimize size of tsbuildinfo

* Deduplicate reference map lists and use file name index to sort them
Different implementation of #43079 based on idea suggested by @sokra

* Minimal json.stringify for the tsbuildinfo
Again implementaion of suggestion by @sokra

* Update src/testRunner/unittests/tsbuild/helpers.ts

Co-authored-by: Tobias Koppers <tobias.koppers@googlemail.com>

* Readable version of buildinfo all the time

* Some renames for readability as per feedback

Co-authored-by: Tobias Koppers <tobias.koppers@googlemail.com>
  • Loading branch information
@sheetalkamat @sokra
sheetalkamat and sokra authored Mar 10, 2021
1 parent 626e78c commit 6e4456b
Show file tree
Hide file tree
Showing 237 changed files with 14,628 additions and 15,138 deletions.
135 changes: 79 additions & 56 deletions src/compiler/builder.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -690,13 +690,16 @@ namespace ts {
return filterSemanticDiagnotics(diagnostics, state.compilerOptions);
}

export type ProgramBuildInfoDiagnostic = string | [string, readonly ReusableDiagnostic[]];
export type ProgramBuilderInfoFilePendingEmit = [string, BuilderFileEmit];
export type ProgramBuildInfoDiagnostic = number | [fileId: number, diagnostics: readonly ReusableDiagnostic[]];
export type ProgramBuilderInfoFilePendingEmit = [fileId: number, emitKind: BuilderFileEmit];
export type ProgramBuildInfoReferencedMap = [fileId: number, fileIdListId: number][];
export interface ProgramBuildInfo {
fileInfos: MapLike<BuilderState.FileInfo>;
fileNames: readonly string[];
fileInfos: readonly BuilderState.FileInfo[];
options: CompilerOptions;
referencedMap?: MapLike<string[]>;
exportedModulesMap?: MapLike<string[]>;
fileIdsList?: readonly (readonly number[])[];
referencedMap?: ProgramBuildInfoReferencedMap;
exportedModulesMap?: ProgramBuildInfoReferencedMap;
semanticDiagnosticsPerFile?: ProgramBuildInfoDiagnostic[];
affectedFilesPendingEmit?: ProgramBuilderInfoFilePendingEmit[];
}
Expand All @@ -708,66 +711,74 @@ namespace ts {
if (outFile(state.compilerOptions)) return undefined;
const currentDirectory = Debug.checkDefined(state.program).getCurrentDirectory();
const buildInfoDirectory = getDirectoryPath(getNormalizedAbsolutePath(getTsBuildInfoEmitOutputFilePath(state.compilerOptions)!, currentDirectory));
const fileInfos: MapLike<BuilderState.FileInfo> = {};
state.fileInfos.forEach((value, key) => {
const fileNames: string[] = [];
const fileNameToFileId = new Map<string, number>();
let fileIdsList: (readonly number[])[] | undefined;
let fileNamesToFileIdListId: ESMap<string, number> | undefined;
const fileInfos = arrayFrom(state.fileInfos.entries(), ([key, value]) => {
// Ensure fileId
const fileId = toFileId(key);
Debug.assert(fileNames[fileId] === relativeToBuildInfo(key));
const signature = state.currentAffectedFilesSignatures && state.currentAffectedFilesSignatures.get(key);
fileInfos[relativeToBuildInfo(key)] = signature === undefined ? value : { version: value.version, signature, affectsGlobalScope: value.affectsGlobalScope };
return signature === undefined ? value : { version: value.version, signature, affectsGlobalScope: value.affectsGlobalScope };
});

const result: ProgramBuildInfo = {
fileInfos,
options: convertToReusableCompilerOptions(state.compilerOptions, relativeToBuildInfoEnsuringAbsolutePath)
};
let referencedMap: ProgramBuildInfoReferencedMap | undefined;
if (state.referencedMap) {
const referencedMap: MapLike<string[]> = {};
for (const key of arrayFrom(state.referencedMap.keys()).sort(compareStringsCaseSensitive)) {
referencedMap[relativeToBuildInfo(key)] = arrayFrom(state.referencedMap.get(key)!.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
}
result.referencedMap = referencedMap;
referencedMap = arrayFrom(state.referencedMap.keys()).sort(compareStringsCaseSensitive).map(key => [
toFileId(key),
toFileIdListId(state.referencedMap!.get(key)!)
]);
}

let exportedModulesMap: ProgramBuildInfoReferencedMap | undefined;
if (state.exportedModulesMap) {
const exportedModulesMap: MapLike<string[]> = {};
for (const key of arrayFrom(state.exportedModulesMap.keys()).sort(compareStringsCaseSensitive)) {
exportedModulesMap = mapDefined(arrayFrom(state.exportedModulesMap.keys()).sort(compareStringsCaseSensitive), key => {
const newValue = state.currentAffectedFilesExportedModulesMap && state.currentAffectedFilesExportedModulesMap.get(key);
// Not in temporary cache, use existing value
if (newValue === undefined) exportedModulesMap[relativeToBuildInfo(key)] = arrayFrom(state.exportedModulesMap.get(key)!.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
if (newValue === undefined) return [toFileId(key), toFileIdListId(state.exportedModulesMap!.get(key)!)];
// Value in cache and has updated value map, use that
else if (newValue) exportedModulesMap[relativeToBuildInfo(key)] = arrayFrom(newValue.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
}
result.exportedModulesMap = exportedModulesMap;
else if (newValue) return [toFileId(key), toFileIdListId(newValue)];
});
}

let semanticDiagnosticsPerFile: ProgramBuildInfoDiagnostic[] | undefined;
if (state.semanticDiagnosticsPerFile) {
const semanticDiagnosticsPerFile: ProgramBuildInfoDiagnostic[] = [];
for (const key of arrayFrom(state.semanticDiagnosticsPerFile.keys()).sort(compareStringsCaseSensitive)) {
const value = state.semanticDiagnosticsPerFile.get(key)!;
semanticDiagnosticsPerFile.push(
(semanticDiagnosticsPerFile ||= []).push(
value.length ?
[
relativeToBuildInfo(key),
toFileId(key),
state.hasReusableDiagnostic ?
value as readonly ReusableDiagnostic[] :
convertToReusableDiagnostics(value as readonly Diagnostic[], relativeToBuildInfo)
] :
relativeToBuildInfo(key)
toFileId(key)
);
}
result.semanticDiagnosticsPerFile = semanticDiagnosticsPerFile;
}

let affectedFilesPendingEmit: ProgramBuilderInfoFilePendingEmit[] | undefined;
if (state.affectedFilesPendingEmit) {
const affectedFilesPendingEmit: ProgramBuilderInfoFilePendingEmit[] = [];
const seenFiles = new Set<Path>();
for (const path of state.affectedFilesPendingEmit.slice(state.affectedFilesPendingEmitIndex).sort(compareStringsCaseSensitive)) {
if (tryAddToSet(seenFiles, path)) {
affectedFilesPendingEmit.push([relativeToBuildInfo(path), state.affectedFilesPendingEmitKind!.get(path)!]);
(affectedFilesPendingEmit ||= []).push([toFileId(path), state.affectedFilesPendingEmitKind!.get(path)!]);
}
}
result.affectedFilesPendingEmit = affectedFilesPendingEmit;
}

return result;
return {
fileNames,
fileInfos,
options: convertToReusableCompilerOptions(state.compilerOptions, relativeToBuildInfoEnsuringAbsolutePath),
fileIdsList,
referencedMap,
exportedModulesMap,
semanticDiagnosticsPerFile,
affectedFilesPendingEmit,
};

function relativeToBuildInfoEnsuringAbsolutePath(path: string) {
return relativeToBuildInfo(getNormalizedAbsolutePath(path, currentDirectory));
Expand All @@ -776,6 +787,22 @@ namespace ts {
function relativeToBuildInfo(path: string) {
return ensurePathIsNonModuleName(getRelativePathFromDirectory(buildInfoDirectory, path, getCanonicalFileName));
}

function toFileId(path: Path): number {
const existing = fileNameToFileId.get(path);
if (existing !== undefined) return existing;
fileNameToFileId.set(path, fileNames.length);
return fileNames.push(relativeToBuildInfo(path)) - 1;
}

function toFileIdListId(set: ReadonlySet<Path>): number {
const fileIds = arrayFrom(set.keys(), toFileId).sort(compareValues);
const key = fileIds.join();
const existing = fileNamesToFileIdListId?.get(key);
if (existing !== undefined) return existing;
(fileNamesToFileIdListId ||= new Map()).set(key, fileIdsList?.length || 0);
return (fileIdsList ||= []).push(fileIds) - 1;
}
}

function convertToReusableCompilerOptions(options: CompilerOptions, relativeToBuildInfo: (path: string) => string) {
Expand Down Expand Up @@ -1167,39 +1194,23 @@ namespace ts {
}
}

function getMapOfReferencedSet(mapLike: MapLike<readonly string[]> | undefined, toPath: (path: string) => Path): ReadonlyESMap<Path, BuilderState.ReferencedSet> | undefined {
if (!mapLike) return undefined;
const map = new Map<Path, BuilderState.ReferencedSet>();
// Copies keys/values from template. Note that for..in will not throw if
// template is undefined, and instead will just exit the loop.
for (const key in mapLike) {
if (hasProperty(mapLike, key)) {
map.set(toPath(key), new Set(mapLike[key].map(toPath)));
}
}
return map;
}

export function createBuildProgramUsingProgramBuildInfo(program: ProgramBuildInfo, buildInfoPath: string, host: ReadBuildProgramHost): EmitAndSemanticDiagnosticsBuilderProgram {
const buildInfoDirectory = getDirectoryPath(getNormalizedAbsolutePath(buildInfoPath, host.getCurrentDirectory()));
const getCanonicalFileName = createGetCanonicalFileName(host.useCaseSensitiveFileNames());

const filePaths = program.fileNames.map(toPath);
const filePathsSetList = program.fileIdsList?.map(fileIds => new Set(fileIds.map(toFilePath)));
const fileInfos = new Map<Path, BuilderState.FileInfo>();
for (const key in program.fileInfos) {
if (hasProperty(program.fileInfos, key)) {
fileInfos.set(toPath(key), program.fileInfos[key]);
}
}

program.fileInfos.forEach((fileInfo, fileId) => fileInfos.set(toFilePath(fileId), fileInfo));
const state: ReusableBuilderProgramState = {
fileInfos,
compilerOptions: convertToOptionsWithAbsolutePaths(program.options, toAbsolutePath),
referencedMap: getMapOfReferencedSet(program.referencedMap, toPath),
exportedModulesMap: getMapOfReferencedSet(program.exportedModulesMap, toPath),
semanticDiagnosticsPerFile: program.semanticDiagnosticsPerFile && arrayToMap(program.semanticDiagnosticsPerFile, value => toPath(isString(value) ? value : value[0]), value => isString(value) ? emptyArray : value[1]),
referencedMap: toMapOfReferencedSet(program.referencedMap),
exportedModulesMap: toMapOfReferencedSet(program.exportedModulesMap),
semanticDiagnosticsPerFile: program.semanticDiagnosticsPerFile && arrayToMap(program.semanticDiagnosticsPerFile, value => toFilePath(isNumber(value) ? value : value[0]), value => isNumber(value) ? emptyArray : value[1]),
hasReusableDiagnostic: true,
affectedFilesPendingEmit: map(program.affectedFilesPendingEmit, value => toPath(value[0])),
affectedFilesPendingEmitKind: program.affectedFilesPendingEmit && arrayToMap(program.affectedFilesPendingEmit, value => toPath(value[0]), value => value[1]),
affectedFilesPendingEmit: map(program.affectedFilesPendingEmit, value => toFilePath(value[0])),
affectedFilesPendingEmitKind: program.affectedFilesPendingEmit && arrayToMap(program.affectedFilesPendingEmit, value => toFilePath(value[0]), value => value[1]),
affectedFilesPendingEmitIndex: program.affectedFilesPendingEmit && 0,
};
return {
Expand Down Expand Up @@ -1234,6 +1245,18 @@ namespace ts {
function toAbsolutePath(path: string) {
return getNormalizedAbsolutePath(path, buildInfoDirectory);
}

function toFilePath(fileId: number) {
return filePaths[fileId];
}

function toFilePathsSet(fileIdsListId: number) {
return filePathsSetList![fileIdsListId];
}

function toMapOfReferencedSet(referenceMap: ProgramBuildInfoReferencedMap | undefined): ReadonlyESMap<Path, BuilderState.ReferencedSet> | undefined {
return referenceMap && arrayToMap(referenceMap, value => toFilePath(value[0]), value => toFilePathsSet(value[1]));
}
}

export function createRedirectedBuilderProgram(state: { program: Program | undefined; compilerOptions: CompilerOptions; }, configFileParsingDiagnostics: readonly Diagnostic[]): BuilderProgram {
Expand Down
2 changes: 1 addition & 1 deletion src/compiler/emitter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -663,7 +663,7 @@ namespace ts {

/*@internal*/
export function getBuildInfoText(buildInfo: BuildInfo) {
return JSON.stringify(buildInfo, undefined, 2);
return JSON.stringify(buildInfo);
}

/*@internal*/
Expand Down
73 changes: 66 additions & 7 deletions src/testRunner/unittests/tsbuild/helpers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,18 +236,78 @@ interface Symbol {
}
}

function generateBuildInfoProgramBaseline(sys: System, originalWriteFile: System["writeFile"], buildInfoPath: string, buildInfo: BuildInfo) {
type ProgramBuildInfoDiagnostic = string | [string, readonly ReusableDiagnostic[]];
type ProgramBuilderInfoFilePendingEmit = [string, BuilderFileEmit];
interface ProgramBuildInfo {
fileInfos: MapLike<BuilderState.FileInfo>;
options: CompilerOptions;
referencedMap?: MapLike<string[]>;
exportedModulesMap?: MapLike<string[]>;
semanticDiagnosticsPerFile?: ProgramBuildInfoDiagnostic[];
affectedFilesPendingEmit?: ProgramBuilderInfoFilePendingEmit[];
}
const fileInfos: ProgramBuildInfo["fileInfos"] = {};
buildInfo.program?.fileInfos.forEach((fileInfo, fileId) => {
fileInfos[toFileName(fileId)] = fileInfo;
});
const fileNamesList = buildInfo.program?.fileIdsList?.map(fileIdsListId => fileIdsListId.map(toFileName));
const program: ProgramBuildInfo | undefined = buildInfo.program && {
fileInfos,
options: buildInfo.program.options,
referencedMap: toMapOfReferencedSet(buildInfo.program.referencedMap),
exportedModulesMap: toMapOfReferencedSet(buildInfo.program.exportedModulesMap),
semanticDiagnosticsPerFile: buildInfo.program.semanticDiagnosticsPerFile?.map(d =>
isNumber(d) ?
toFileName(d) :
[toFileName(d[0]), d[1]]
),
affectedFilesPendingEmit: buildInfo.program.affectedFilesPendingEmit?.map(([fileId, emitKind]) => [
toFileName(fileId),
emitKind
]),
};
const result: Omit<BuildInfo, "program"> & { program: ProgramBuildInfo | undefined; } = {
bundle: buildInfo.bundle,
program,
version: buildInfo.version === version ? fakes.version : buildInfo.version,
};
// For now its just JSON.stringify
originalWriteFile.call(sys, `${buildInfoPath}.readable.baseline.txt`, JSON.stringify(result, /*replacer*/ undefined, 2));

function toFileName(fileId: number) {
return buildInfo.program!.fileNames[fileId];
}

function toFileNames(fileIdsListId: number) {
return fileNamesList![fileIdsListId];
}

function toMapOfReferencedSet(referenceMap: ProgramBuildInfoReferencedMap | undefined): MapLike<string[]> | undefined {
if (!referenceMap) return undefined;
const result: MapLike<string[]> = {};
for (const [fileNamesKey, fileNamesListKey] of referenceMap) {
result[toFileName(fileNamesKey)] = toFileNames(fileNamesListKey);
}
return result;
}
}

export function baselineBuildInfo(
options: CompilerOptions,
sys: System & { writtenFiles: ReadonlyCollection<string>; },
originalReadCall?: System["readFile"]
originalReadCall?: System["readFile"],
originalWriteFile?: System["writeFile"],
) {
const out = outFile(options);
if (!out) return;
const { buildInfoPath, jsFilePath, declarationFilePath } = getOutputPathsForBundle(options, /*forceDts*/ false);
const buildInfoPath = getTsBuildInfoEmitOutputFilePath(options);
if (!buildInfoPath || !sys.writtenFiles.has(buildInfoPath)) return;
if (!sys.fileExists(buildInfoPath)) return;

const buildInfo = getBuildInfo((originalReadCall || sys.readFile).call(sys, buildInfoPath, "utf8")!);
generateBuildInfoProgramBaseline(sys, originalWriteFile || sys.writeFile, buildInfoPath, buildInfo);

if (!outFile(options)) return;
const { jsFilePath, declarationFilePath } = getOutputPathsForBundle(options, /*forceDts*/ false);
const bundle = buildInfo.bundle;
if (!bundle || (!length(bundle.js && bundle.js.sections) && !length(bundle.dts && bundle.dts.sections))) return;

Expand All @@ -256,9 +316,8 @@ interface Symbol {
generateBundleFileSectionInfo(sys, originalReadCall || sys.readFile, baselineRecorder, bundle.js, jsFilePath);
generateBundleFileSectionInfo(sys, originalReadCall || sys.readFile, baselineRecorder, bundle.dts, declarationFilePath);
baselineRecorder.Close();

const text = baselineRecorder.lines.join("\r\n");
sys.writeFile(`${buildInfoPath}.baseline.txt`, text);
(originalWriteFile || sys.writeFile).call(sys, `${buildInfoPath}.baseline.txt`, text);
}

interface VerifyIncrementalCorrectness {
Expand Down Expand Up @@ -295,7 +354,7 @@ interface Symbol {
const cleanBuildText = sys.readFile(outputFile);
const incrementalBuildText = newSys.readFile(outputFile);
const descrepancyInClean = discrepancies?.get(outputFile);
if (!isBuildInfoFile(outputFile)) {
if (!isBuildInfoFile(outputFile) && !fileExtensionIs(outputFile, ".tsbuildinfo.readable.baseline.txt")) {
verifyTextEqual(incrementalBuildText, cleanBuildText, descrepancyInClean, `File: ${outputFile}`);
}
else if (incrementalBuildText !== cleanBuildText) {
Expand Down
9 changes: 4 additions & 5 deletions src/testRunner/unittests/tsbuild/outputPaths.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,11 +60,10 @@ namespace ts {
noChangeRun,
{
...noChangeProject,
cleanBuildDiscrepancies: () => {
const map = new Map<string, CleanBuildDescrepancy>();
map.set("/src/dist/tsconfig.tsbuildinfo", CleanBuildDescrepancy.CleanFileTextDifferent); // tsbuildinfo will have -p setting when built using -p vs no build happens incrementally because of no change.
return map;
}
cleanBuildDiscrepancies: () => new Map([
["/src/dist/tsconfig.tsbuildinfo", CleanBuildDescrepancy.CleanFileTextDifferent], // tsbuildinfo will have -p setting when built using -p vs no build happens incrementally because of no change.
["/src/dist/tsconfig.tsbuildinfo.readable.baseline.txt", CleanBuildDescrepancy.CleanFileTextDifferent] // tsbuildinfo will have -p setting when built using -p vs no build happens incrementally because of no change.
]),
}
],
}, ["/src/dist/src/index.js", "/src/dist/src/index.d.ts"]);
Expand Down
Loading

0 comments on commit 6e4456b

Please sign in to comment.