Build improvements (#48784)

* Use fixed time for vfs so baselining is consistent

* Baseline buildinfos

* Write new file text in baseline even if the file wasnt read on the shadow

* Remove unnecessary debugger statement

* Make sure that incremental correctness is checked with correct writeFile so we know buildInfo was written
Also baseline these so its easy to verify the changes

* More baselines for the tsbuildinfo

* If we are writing dts file and have used file text as version, we can update the signature when doing actual emit

* Make WriteFileCallback Api ready for future

* Assert that there is only single source file when emitting d.ts file

* Add test

* Renames

* More refactoring

* If we are updating dts of any of the file and it affects global scope, everything needs update in signature and dts emit
Fixes #42769

* Stacktrace optimization for getModified time in anticipation of using it more than fileExists wherever possible

* Baseline getModifiedTime, setModifiedTime, fileExits and directoryExits for experiment

* Remove unnecessary write file finger print code since its not used at all

* Use modified time instead of file existence check

* Remove unnecessary getModifiedTime

* No need to check for file existence before reading the d.ts file

* Do project reference errors before doing input/output file checks

* Dont call getModifiedTimes if dts change

* Passdown modified time if queried

* Use modified time passed through the file watching in tsbuild

* Handle force build as separate upto date status

* uptodate status worker to read buildinfo and use it to determine upto date ness

* No need to update output timestamps if buildinfo will determine uptodateness

* Store change file set instead of hasPendingChange to be able to reuse the information

* Add test that shows input file is not present

* No need to check input time stamp before buildinfo

* Keep buildinfos for lifetime of the solution builder and project

* Store modified time along with text of buildinfo

* Non composite projects dont need to track declaration change time

* Pass through buildInfo so we dont have to parse it back

* Save dts change time in buildinfo itself

* Store dts time for --out in the buildInfo

* Store hash of text in the bundle info so it can be verified before manipulating text for fast updates during prepend
This helps when text changes during incremental build toggling and we determine we can just manipulate text

* Since buildinfo is cached no need to maintain version check state

* Store output time stamps for non incremental builds

* Revert "Baseline getModifiedTime, setModifiedTime, fileExits and directoryExits for experiment"

This reverts commit 7e65cd3.

* Change verbose messages for upto date status

* Reconcile reusable builder state and builder state so there are not two different types that are almost similar looking

* Cleanup impliedFormat

* Cleanup

* Cleanup noEmit option

* BuildInfo options emit as a flag

* Factor out types for program written in buildinfo with and without bundle emit

* No need to store output file stamps if not in watch mode

* Cleanup

* Test for single watch per file

* Fix emit and error update baselines that were duplicate

* More refactoring

* Only copy emit state fields when backing up to restore if emit fails

* Instead of maintaining delta of changes, maintain old state for those changes

* Add test to verify build when input file does not change

* If version of the input file does not change, dont mark as out of date

* Disable lint warning as build fails without the assert

* Report aggregate statistics for solution as well as some solution perf numbers

* Options solutionDiagnostics instead so that its not too verbose when printing diagnostics

* When tsc --build --clean, only remove tsbuildinfo if its incremental build

* Revert "Options solutionDiagnostics instead so that its not too verbose when printing diagnostics"

This reverts commit 0cf9e30.

* Revert "Report aggregate statistics for solution as well as some solution perf numbers"

This reverts commit 405d8e9.

* Revert "When tsc --build --clean, only remove tsbuildinfo if its incremental build"

This reverts commit e4e6672.

* Comments in the code

* Feedback

Author: sheetalkamat

src/compiler/builder.ts

@@ -25,6 +25,11 @@ namespace ts {
          */
         /*@internal*/
         storeFilesChangingSignatureDuringEmit?: boolean;
+        /**
+         * Gets the current time
+         */
+        /*@internal*/
+        now?(): Date;
     }
 
     /**
@@ -34,9 +39,9 @@ namespace ts {
         /*@internal*/
         getState(): ReusableBuilderProgramState;
         /*@internal*/
-        backupState(): void;
+        saveEmitState(): SavedBuildProgramEmitState;
         /*@internal*/
-        restoreState(): void;
+        restoreEmitState(saved: SavedBuildProgramEmitState): void;
         /**
          * Returns current program
          */

src/compiler/builderPublic.ts

@@ -10,5 +10,6 @@ namespace ts {
         name: string;
         writeByteOrderMark: boolean;
         text: string;
+        /* @internal */ buildInfo?: BuildInfo
     }
 }

src/compiler/builderState.ts

@@ -5140,11 +5140,11 @@
         "category": "Error",
         "code": 6310
     },
-    "Project '{0}' is out of date because oldest output '{1}' is older than newest input '{2}'": {
+    "Project '{0}' is out of date because output '{1}' is older than input '{2}'": {
         "category": "Message",
         "code": 6350
     },
-    "Project '{0}' is up to date because newest input '{1}' is older than oldest output '{2}'": {
+    "Project '{0}' is up to date because newest input '{1}' is older than output '{2}'": {
         "category": "Message",
         "code": 6351
     },
@@ -5322,6 +5322,14 @@
         "category": "Message",
         "code": 6398
     },
+    "Project '{0}' is out of date because buildinfo file '{1}' indicates that some of the changes were not emitted": {
+        "category": "Message",
+        "code": 6399
+    },
+    "Project '{0}' is up to date but needs to update timestamps of output files that are older than input files": {
+        "category": "Message",
+        "code": 6400
+    },
 
     "The expected type comes from property '{0}' which is declared here on type '{1}'": {
         "category": "Message",

src/compiler/builderStatePublic.ts

@@ -366,7 +366,9 @@ namespace ts {
                 return;
             }
             const version = ts.version; // Extracted into a const so the form is stable between namespace and module
-            writeFile(host, emitterDiagnostics, buildInfoPath, getBuildInfoText({ bundle, program, version }), /*writeByteOrderMark*/ false);
+            const buildInfo: BuildInfo = { bundle, program, version };
+            // Pass buildinfo as additional data to avoid having to reparse
+            writeFile(host, emitterDiagnostics, buildInfoPath, getBuildInfoText(buildInfo), /*writeByteOrderMark*/ false, /*sourceFiles*/ undefined, { buildInfo });
         }
 
         function emitJsFileOrBundle(
@@ -557,14 +559,19 @@ namespace ts {
                 if (sourceMapFilePath) {
                     const sourceMap = sourceMapGenerator.toString();
                     writeFile(host, emitterDiagnostics, sourceMapFilePath, sourceMap, /*writeByteOrderMark*/ false, sourceFiles);
+                    if (printer.bundleFileInfo) printer.bundleFileInfo.mapHash = BuilderState.computeSignature(sourceMap, maybeBind(host, host.createHash));
                 }
             }
             else {
                 writer.writeLine();
             }
 
             // Write the output file
-            writeFile(host, emitterDiagnostics, jsFilePath, writer.getText(), !!compilerOptions.emitBOM, sourceFiles, { sourceMapUrlPos });
+            const text = writer.getText();
+            writeFile(host, emitterDiagnostics, jsFilePath, text, !!compilerOptions.emitBOM, sourceFiles, { sourceMapUrlPos });
+            // We store the hash of the text written in the buildinfo to ensure that text of the referenced d.ts file is same as whats in the buildinfo
+            // This is needed because incremental can be toggled between two runs and we might use stale file text to do text manipulation in prepend mode
+            if (printer.bundleFileInfo) printer.bundleFileInfo.hash = BuilderState.computeSignature(text, maybeBind(host, host.createHash));
 
             // Reset state
             writer.clear();
@@ -709,6 +716,9 @@ namespace ts {
         getCanonicalFileName(fileName: string): string;
         useCaseSensitiveFileNames(): boolean;
         getNewLine(): string;
+        createHash?(data: string): string;
+        now?(): Date;
+        getBuildInfo?(fileName: string, configFilePath: string | undefined): BuildInfo | undefined;
     }
 
     function createSourceFilesFromBundleBuildInfo(bundle: BundleBuildInfo, buildInfoDirectory: string, host: EmitUsingBuildInfoHost): readonly SourceFile[] {
@@ -745,23 +755,40 @@ namespace ts {
         getCommandLine: (ref: ProjectReference) => ParsedCommandLine | undefined,
         customTransformers?: CustomTransformers
     ): EmitUsingBuildInfoResult {
+        const createHash = maybeBind(host, host.createHash);
         const { buildInfoPath, jsFilePath, sourceMapFilePath, declarationFilePath, declarationMapPath } = getOutputPathsForBundle(config.options, /*forceDtsPaths*/ false);
-        const buildInfoText = host.readFile(Debug.checkDefined(buildInfoPath));
-        if (!buildInfoText) return buildInfoPath!;
+        let buildInfo: BuildInfo;
+        if (host.getBuildInfo) {
+            // If host directly provides buildinfo we can get it directly. This allows host to cache the buildinfo
+            const hostBuildInfo = host.getBuildInfo(buildInfoPath!, config.options.configFilePath);
+            if (!hostBuildInfo) return buildInfoPath!;
+            buildInfo = hostBuildInfo;
+        }
+        else {
+            const buildInfoText = host.readFile(buildInfoPath!);
+            if (!buildInfoText) return buildInfoPath!;
+            buildInfo = getBuildInfo(buildInfoText);
+        }
+        if (!buildInfo.bundle || !buildInfo.bundle.js || (declarationFilePath && !buildInfo.bundle.dts)) return buildInfoPath!;
+
         const jsFileText = host.readFile(Debug.checkDefined(jsFilePath));
         if (!jsFileText) return jsFilePath!;
+        // If the jsFileText is not same has what it was created with, tsbuildinfo is stale so dont use it
+        if (BuilderState.computeSignature(jsFileText, createHash) !== buildInfo.bundle.js.hash) return jsFilePath!;
         const sourceMapText = sourceMapFilePath && host.readFile(sourceMapFilePath);
         // error if no source map or for now if inline sourcemap
         if ((sourceMapFilePath && !sourceMapText) || config.options.inlineSourceMap) return sourceMapFilePath || "inline sourcemap decoding";
+        if (sourceMapFilePath && BuilderState.computeSignature(sourceMapText!, createHash) !== buildInfo.bundle.js.mapHash) return sourceMapFilePath;
+
         // read declaration text
         const declarationText = declarationFilePath && host.readFile(declarationFilePath);
         if (declarationFilePath && !declarationText) return declarationFilePath;
+        if (declarationFilePath && BuilderState.computeSignature(declarationText!, createHash) !== buildInfo.bundle.dts!.hash) return declarationFilePath;
         const declarationMapText = declarationMapPath && host.readFile(declarationMapPath);
         // error if no source map or for now if inline sourcemap
         if ((declarationMapPath && !declarationMapText) || config.options.inlineSourceMap) return declarationMapPath || "inline sourcemap decoding";
+        if (declarationMapPath && BuilderState.computeSignature(declarationMapText!, createHash) !== buildInfo.bundle.dts!.mapHash) return declarationMapPath;
 
-        const buildInfo = getBuildInfo(buildInfoText);
-        if (!buildInfo.bundle || !buildInfo.bundle.js || (declarationText && !buildInfo.bundle.dts)) return buildInfoPath!;
         const buildInfoDirectory = getDirectoryPath(getNormalizedAbsolutePath(buildInfoPath!, host.getCurrentDirectory()));
         const ownPrependInput = createInputFiles(
             jsFileText,
@@ -779,6 +806,8 @@ namespace ts {
         const outputFiles: OutputFile[] = [];
         const prependNodes = createPrependNodes(config.projectReferences, getCommandLine, f => host.readFile(f));
         const sourceFilesForJsEmit = createSourceFilesFromBundleBuildInfo(buildInfo.bundle, buildInfoDirectory, host);
+        let changedDtsText: string | undefined;
+        let changedDtsData: WriteFileCallbackData | undefined;
         const emitHost: EmitHost = {
             getPrependNodes: memoize(() => [...prependNodes, ownPrependInput]),
             getCanonicalFileName: host.getCanonicalFileName,
@@ -794,7 +823,7 @@ namespace ts {
             getResolvedProjectReferenceToRedirect: returnUndefined,
             getProjectReferenceRedirect: returnUndefined,
             isSourceOfProjectReferenceRedirect: returnFalse,
-            writeFile: (name, text, writeByteOrderMark) => {
+            writeFile: (name, text, writeByteOrderMark, _onError, _sourceFiles, data) => {
                 switch (name) {
                     case jsFilePath:
                         if (jsFileText === text) return;
@@ -803,19 +832,26 @@ namespace ts {
                         if (sourceMapText === text) return;
                         break;
                     case buildInfoPath:
-                        const newBuildInfo = getBuildInfo(text);
+                        const newBuildInfo = data!.buildInfo!;
                         newBuildInfo.program = buildInfo.program;
+                        if (newBuildInfo.program && changedDtsText !== undefined && config.options.composite) {
+                            // Update the output signature
+                            (newBuildInfo.program as ProgramBundleEmitBuildInfo).outSignature = computeSignature(changedDtsText, changedDtsData, createHash);
+                            newBuildInfo.program.dtsChangeTime = getCurrentTime(host).getTime();
+                        }
                         // Update sourceFileInfo
                         const { js, dts, sourceFiles } = buildInfo.bundle!;
                         newBuildInfo.bundle!.js!.sources = js!.sources;
                         if (dts) {
                             newBuildInfo.bundle!.dts!.sources = dts.sources;
                         }
                         newBuildInfo.bundle!.sourceFiles = sourceFiles;
-                        outputFiles.push({ name, text: getBuildInfoText(newBuildInfo), writeByteOrderMark });
+                        outputFiles.push({ name, text: getBuildInfoText(newBuildInfo), writeByteOrderMark, buildInfo: newBuildInfo });
                         return;
                     case declarationFilePath:
                         if (declarationText === text) return;
+                        changedDtsText = text;
+                        changedDtsData = data;
                         break;
                     case declarationMapPath:
                         if (declarationMapText === text) return;
@@ -833,6 +869,7 @@ namespace ts {
             getSourceFileFromReference: returnUndefined,
             redirectTargetsMap: createMultiMap(),
             getFileIncludeReasons: notImplemented,
+            createHash,
         };
         emitFiles(
             notImplementedResolver,

src/compiler/commandLineParser.ts

@@ -58,22 +58,14 @@ namespace ts {
         return getPathFromPathComponents(commonPathComponents);
     }
 
-    interface OutputFingerprint {
-        hash: string;
-        byteOrderMark: boolean;
-        mtime: Date;
-    }
-
     export function createCompilerHost(options: CompilerOptions, setParentNodes?: boolean): CompilerHost {
         return createCompilerHostWorker(options, setParentNodes);
     }
 
     /*@internal*/
-    // TODO(shkamat): update this after reworking ts build API
     export function createCompilerHostWorker(options: CompilerOptions, setParentNodes?: boolean, system = sys): CompilerHost {
         const existingDirectories = new Map<string, boolean>();
         const getCanonicalFileName = createGetCanonicalFileName(system.useCaseSensitiveFileNames);
-        const computeHash = maybeBind(system, system.createHash) || generateDjb2Hash;
         function getSourceFile(fileName: string, languageVersionOrOptions: ScriptTarget | CreateSourceFileOptions, onError?: (message: string) => void): SourceFile | undefined {
             let text: string | undefined;
             try {
@@ -113,7 +105,7 @@ namespace ts {
                     fileName,
                     data,
                     writeByteOrderMark,
-                    (path, data, writeByteOrderMark) => writeFileWorker(path, data, writeByteOrderMark),
+                    (path, data, writeByteOrderMark) => system.writeFile(path, data, writeByteOrderMark),
                     path => (compilerHost.createDirectory || system.createDirectory)(path),
                     path => directoryExists(path));
 
@@ -127,42 +119,6 @@ namespace ts {
             }
         }
 
-        let outputFingerprints: ESMap<string, OutputFingerprint>;
-        function writeFileWorker(fileName: string, data: string, writeByteOrderMark: boolean) {
-            if (!isWatchSet(options) || !system.getModifiedTime) {
-                system.writeFile(fileName, data, writeByteOrderMark);
-                return;
-            }
-
-            if (!outputFingerprints) {
-                outputFingerprints = new Map<string, OutputFingerprint>();
-            }
-
-            const hash = computeHash(data);
-            const mtimeBefore = system.getModifiedTime(fileName);
-
-            if (mtimeBefore) {
-                const fingerprint = outputFingerprints.get(fileName);
-                // If output has not been changed, and the file has no external modification
-                if (fingerprint &&
-                    fingerprint.byteOrderMark === writeByteOrderMark &&
-                    fingerprint.hash === hash &&
-                    fingerprint.mtime.getTime() === mtimeBefore.getTime()) {
-                    return;
-                }
-            }
-
-            system.writeFile(fileName, data, writeByteOrderMark);
-
-            const mtimeAfter = system.getModifiedTime(fileName) || missingFileModifiedTime;
-
-            outputFingerprints.set(fileName, {
-                hash,
-                byteOrderMark: writeByteOrderMark,
-                mtime: mtimeAfter
-            });
-        }
-
         function getDefaultLibLocation(): string {
             return getDirectoryPath(normalizePath(system.getExecutingFilePath()));
         }
@@ -1973,6 +1929,7 @@ namespace ts {
                 getSourceFileFromReference: (file, ref) => program.getSourceFileFromReference(file, ref),
                 redirectTargetsMap,
                 getFileIncludeReasons: program.getFileIncludeReasons,
+                createHash: maybeBind(host, host.createHash),
             };
         }