diff --git a/src/compiler/moduleNameResolver.ts b/src/compiler/moduleNameResolver.ts
index 28550ab8cd103..82e39ecfad460 100644
--- a/src/compiler/moduleNameResolver.ts
+++ b/src/compiler/moduleNameResolver.ts
@@ -84,7 +84,11 @@ namespace ts {
         return { fileName: resolved.path, packageId: resolved.packageId };
     }
 
-    function createResolvedModuleWithFailedLookupLocations(resolved: Resolved | undefined, isExternalLibraryImport: boolean, failedLookupLocations: string[]): ResolvedModuleWithFailedLookupLocations {
+    function createResolvedModuleWithFailedLookupLocations(resolved: Resolved | undefined, isExternalLibraryImport: boolean | undefined, failedLookupLocations: string[], resultFromCache: ResolvedModuleWithFailedLookupLocations | undefined): ResolvedModuleWithFailedLookupLocations {
+        if (resultFromCache) {
+            resultFromCache.failedLookupLocations.push(...failedLookupLocations);
+            return resultFromCache;
+        }
         return {
             resolvedModule: resolved && { resolvedFileName: resolved.path, originalPath: resolved.originalPath === true ? undefined : resolved.originalPath, extension: resolved.extension, isExternalLibraryImport, packageId: resolved.packageId },
             failedLookupLocations
@@ -96,6 +100,7 @@ namespace ts {
         compilerOptions: CompilerOptions;
         traceEnabled: boolean;
         failedLookupLocations: Push<string>;
+        resultFromCache?: ResolvedModuleWithFailedLookupLocations;
     }
 
     /** Just the fields that we use for module resolution. */
@@ -926,11 +931,7 @@ namespace ts {
         const state: ModuleResolutionState = { compilerOptions, host, traceEnabled, failedLookupLocations };
 
         const result = forEach(extensions, ext => tryResolve(ext));
-        if (result && result.value) {
-            const { resolved, isExternalLibraryImport } = result.value;
-            return createResolvedModuleWithFailedLookupLocations(resolved, isExternalLibraryImport, failedLookupLocations);
-        }
-        return { resolvedModule: undefined, failedLookupLocations };
+        return createResolvedModuleWithFailedLookupLocations(result?.value?.resolved, result?.value?.isExternalLibraryImport, failedLookupLocations, state.resultFromCache);
 
         function tryResolve(extensions: Extensions): SearchResult<{ resolved: Resolved, isExternalLibraryImport: boolean }> {
             const loader: ResolutionKindSpecificLoader = (extensions, candidate, onlyRecordFailures, state) => nodeLoadModuleByRelativeName(extensions, candidate, onlyRecordFailures, state, /*considerPackageJson*/ true);
@@ -1435,7 +1436,7 @@ namespace ts {
             if (state.traceEnabled) {
                 trace(state.host, Diagnostics.Resolution_for_module_0_was_found_in_cache_from_location_1, moduleName, containingDirectory);
             }
-            state.failedLookupLocations.push(...result.failedLookupLocations);
+            state.resultFromCache = result;
             return { value: result.resolvedModule && { path: result.resolvedModule.resolvedFileName, originalPath: result.resolvedModule.originalPath || true, extension: result.resolvedModule.extension, packageId: result.resolvedModule.packageId } };
         }
     }
@@ -1448,7 +1449,7 @@ namespace ts {
 
         const resolved = tryResolve(Extensions.TypeScript) || tryResolve(Extensions.JavaScript);
         // No originalPath because classic resolution doesn't resolve realPath
-        return createResolvedModuleWithFailedLookupLocations(resolved && resolved.value, /*isExternalLibraryImport*/ false, failedLookupLocations);
+        return createResolvedModuleWithFailedLookupLocations(resolved && resolved.value, /*isExternalLibraryImport*/ false, failedLookupLocations, state.resultFromCache);
 
         function tryResolve(extensions: Extensions): SearchResult<Resolved> {
             const resolvedUsingSettings = tryLoadModuleUsingOptionalResolutionSettings(extensions, moduleName, containingDirectory, loadModuleFromFileNoPackageId, state);
@@ -1495,7 +1496,7 @@ namespace ts {
         const failedLookupLocations: string[] = [];
         const state: ModuleResolutionState = { compilerOptions, host, traceEnabled, failedLookupLocations };
         const resolved = loadModuleFromImmediateNodeModulesDirectory(Extensions.DtsOnly, moduleName, globalCache, state, /*typesScopeOnly*/ false);
-        return createResolvedModuleWithFailedLookupLocations(resolved, /*isExternalLibraryImport*/ true, failedLookupLocations);
+        return createResolvedModuleWithFailedLookupLocations(resolved, /*isExternalLibraryImport*/ true, failedLookupLocations, state.resultFromCache);
     }
 
     /**
diff --git a/src/compiler/resolutionCache.ts b/src/compiler/resolutionCache.ts
index d512087154473..de26a3ea148f1 100644
--- a/src/compiler/resolutionCache.ts
+++ b/src/compiler/resolutionCache.ts
@@ -25,9 +25,11 @@ namespace ts {
     }
 
     interface ResolutionWithFailedLookupLocations {
-        readonly failedLookupLocations: readonly string[];
+        readonly failedLookupLocations: string[];
         isInvalidated?: boolean;
         refCount?: number;
+        // Files that have this resolution using
+        files?: Path[];
     }
 
     interface ResolutionWithResolvedFileName {
@@ -53,7 +55,6 @@ namespace ts {
         getGlobalCache?(): string | undefined;
         globalCacheResolutionModuleName?(externalModuleName: string): string;
         writeLog(s: string): void;
-        maxNumberOfFilesToIterateForInvalidation?: number;
         getCurrentProgram(): Program | undefined;
         fileIsOpen(filePath: Path): boolean;
     }
@@ -133,8 +134,6 @@ namespace ts {
         return true;
     }
 
-    export const maxNumberOfFilesToIterateForInvalidation = 256;
-
     type GetResolutionWithResolvedFileName<T extends ResolutionWithFailedLookupLocations = ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName = ResolutionWithResolvedFileName> =
         (resolution: T) => R | undefined;
 
@@ -142,9 +141,11 @@ namespace ts {
         let filesWithChangedSetOfUnresolvedImports: Path[] | undefined;
         let filesWithInvalidatedResolutions: Map<true> | undefined;
         let filesWithInvalidatedNonRelativeUnresolvedImports: ReadonlyMap<readonly string[]> | undefined;
-        let allFilesHaveInvalidatedResolution = false;
         const nonRelativeExternalModuleResolutions = createMultiMap<ResolutionWithFailedLookupLocations>();
 
+        const resolutionsWithFailedLookups: ResolutionWithFailedLookupLocations[] = [];
+        const resolvedFileToResolution = createMultiMap<ResolutionWithFailedLookupLocations>();
+
         const getCurrentDirectory = memoize(() => resolutionHost.getCurrentDirectory!()); // TODO: GH#18217
         const cachedDirectoryStructureHost = resolutionHost.getCachedDirectoryStructureHost();
 
@@ -223,7 +224,8 @@ namespace ts {
             closeTypeRootsWatch();
             resolvedModuleNames.clear();
             resolvedTypeReferenceDirectives.clear();
-            allFilesHaveInvalidatedResolution = false;
+            resolvedFileToResolution.clear();
+            resolutionsWithFailedLookups.length = 0;
             // perDirectoryResolvedModuleNames and perDirectoryResolvedTypeReferenceDirectives could be non empty if there was exception during program update
             // (between startCachingPerDirectoryResolution and finishCachingPerDirectoryResolution)
             clearPerDirectoryResolutions();
@@ -250,7 +252,7 @@ namespace ts {
         }
 
         function createHasInvalidatedResolution(forceAllFilesAsInvalidated?: boolean): HasInvalidatedResolution {
-            if (allFilesHaveInvalidatedResolution || forceAllFilesAsInvalidated) {
+            if (forceAllFilesAsInvalidated) {
                 // Any file asked would have invalidated resolution
                 filesWithInvalidatedResolutions = undefined;
                 return returnTrue;
@@ -270,7 +272,6 @@ namespace ts {
         }
 
         function finishCachingPerDirectoryResolution() {
-            allFilesHaveInvalidatedResolution = false;
             filesWithInvalidatedNonRelativeUnresolvedImports = undefined;
             clearPerDirectoryResolutions();
             directoryWatchesOfFailedLookups.forEach((watcher, path) => {
@@ -300,7 +301,10 @@ namespace ts {
                     host,
                     globalCache);
                 if (resolvedModule) {
-                    return { resolvedModule, failedLookupLocations: addRange(primaryResult.failedLookupLocations as string[], failedLookupLocations) };
+                    // Modify existing resolution so its saved in the directory cache as well
+                    (primaryResult.resolvedModule as any) = resolvedModule;
+                    primaryResult.failedLookupLocations.push(...failedLookupLocations);
+                    return primaryResult;
                 }
             }
 
@@ -308,18 +312,24 @@ namespace ts {
             return primaryResult;
         }
 
-        function resolveNamesWithLocalCache<T extends ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName>(
-            names: readonly string[],
-            containingFile: string,
-            redirectedReference: ResolvedProjectReference | undefined,
-            cache: Map<Map<T>>,
-            perDirectoryCacheWithRedirects: CacheWithRedirects<Map<T>>,
-            loader: (name: string, containingFile: string, options: CompilerOptions, host: ModuleResolutionHost, redirectedReference?: ResolvedProjectReference) => T,
-            getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName<T, R>,
-            shouldRetryResolution: (t: T) => boolean,
-            reusedNames: readonly string[] | undefined,
-            logChanges: boolean): (R | undefined)[] {
-
+        interface ResolveNamesWithLocalCacheInput<T extends ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName> {
+            names: readonly string[];
+            containingFile: string;
+            redirectedReference: ResolvedProjectReference | undefined;
+            cache: Map<Map<T>>;
+            perDirectoryCacheWithRedirects: CacheWithRedirects<Map<T>>;
+            loader: (name: string, containingFile: string, options: CompilerOptions, host: ModuleResolutionHost, redirectedReference?: ResolvedProjectReference) => T;
+            getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName<T, R>;
+            shouldRetryResolution: (t: T) => boolean;
+            reusedNames?: readonly string[];
+            logChanges?: boolean;
+        }
+        function resolveNamesWithLocalCache<T extends ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName>({
+            names, containingFile, redirectedReference,
+            cache, perDirectoryCacheWithRedirects,
+            loader, getResolutionWithResolvedFileName,
+            shouldRetryResolution, reusedNames, logChanges
+        }: ResolveNamesWithLocalCacheInput<T, R>): (R | undefined)[] {
             const path = resolutionHost.toPath(containingFile);
             const resolutionsInFile = cache.get(path) || cache.set(path, createMap()).get(path)!;
             const dirPath = getDirectoryPath(path);
@@ -345,7 +355,7 @@ namespace ts {
                 let resolution = resolutionsInFile.get(name);
                 // Resolution is valid if it is present and not invalidated
                 if (!seenNamesInFile.has(name) &&
-                    allFilesHaveInvalidatedResolution || unmatchedRedirects || !resolution || resolution.isInvalidated ||
+                    unmatchedRedirects || !resolution || resolution.isInvalidated ||
                     // If the name is unresolved import that was invalidated, recalculate
                     (hasInvalidatedNonRelativeUnresolvedImport && !isExternalModuleNameRelative(name) && shouldRetryResolution(resolution))) {
                     const existingResolution = resolution;
@@ -358,9 +368,9 @@ namespace ts {
                         perDirectoryResolution.set(name, resolution);
                     }
                     resolutionsInFile.set(name, resolution);
-                    watchFailedLookupLocationsOfExternalModuleResolutions(name, resolution);
+                    watchFailedLookupLocationsOfExternalModuleResolutions(name, resolution, path, getResolutionWithResolvedFileName);
                     if (existingResolution) {
-                        stopWatchFailedLookupLocationOfResolution(existingResolution);
+                        stopWatchFailedLookupLocationOfResolution(existingResolution, path, getResolutionWithResolvedFileName);
                     }
 
                     if (logChanges && filesWithChangedSetOfUnresolvedImports && !resolutionIsEqualTo(existingResolution, resolution)) {
@@ -377,7 +387,7 @@ namespace ts {
             // Stop watching and remove the unused name
             resolutionsInFile.forEach((resolution, name) => {
                 if (!seenNamesInFile.has(name) && !contains(reusedNames, name)) {
-                    stopWatchFailedLookupLocationOfResolution(resolution);
+                    stopWatchFailedLookupLocationOfResolution(resolution, path, getResolutionWithResolvedFileName);
                     resolutionsInFile.delete(name);
                 }
             });
@@ -404,23 +414,31 @@ namespace ts {
         }
 
         function resolveTypeReferenceDirectives(typeDirectiveNames: string[], containingFile: string, redirectedReference?: ResolvedProjectReference): (ResolvedTypeReferenceDirective | undefined)[] {
-            return resolveNamesWithLocalCache<CachedResolvedTypeReferenceDirectiveWithFailedLookupLocations, ResolvedTypeReferenceDirective>(
-                typeDirectiveNames, containingFile, redirectedReference,
-                resolvedTypeReferenceDirectives, perDirectoryResolvedTypeReferenceDirectives,
-                resolveTypeReferenceDirective, getResolvedTypeReferenceDirective,
-                /*shouldRetryResolution*/ resolution => resolution.resolvedTypeReferenceDirective === undefined,
-                /*reusedNames*/ undefined, /*logChanges*/ false
-            );
+            return resolveNamesWithLocalCache<CachedResolvedTypeReferenceDirectiveWithFailedLookupLocations, ResolvedTypeReferenceDirective>({
+                names: typeDirectiveNames,
+                containingFile,
+                redirectedReference,
+                cache: resolvedTypeReferenceDirectives,
+                perDirectoryCacheWithRedirects: perDirectoryResolvedTypeReferenceDirectives,
+                loader: resolveTypeReferenceDirective,
+                getResolutionWithResolvedFileName: getResolvedTypeReferenceDirective,
+                shouldRetryResolution: resolution => resolution.resolvedTypeReferenceDirective === undefined,
+            });
         }
 
         function resolveModuleNames(moduleNames: string[], containingFile: string, reusedNames: string[] | undefined, redirectedReference?: ResolvedProjectReference): (ResolvedModuleFull | undefined)[] {
-            return resolveNamesWithLocalCache<CachedResolvedModuleWithFailedLookupLocations, ResolvedModuleFull>(
-                moduleNames, containingFile, redirectedReference,
-                resolvedModuleNames, perDirectoryResolvedModuleNames,
-                resolveModuleName, getResolvedModule,
-                /*shouldRetryResolution*/ resolution => !resolution.resolvedModule || !resolutionExtensionIsTSOrJson(resolution.resolvedModule.extension),
-                reusedNames, logChangesWhenResolvingModule
-            );
+            return resolveNamesWithLocalCache<CachedResolvedModuleWithFailedLookupLocations, ResolvedModuleFull>({
+                names: moduleNames,
+                containingFile,
+                redirectedReference,
+                cache: resolvedModuleNames,
+                perDirectoryCacheWithRedirects: perDirectoryResolvedModuleNames,
+                loader: resolveModuleName,
+                getResolutionWithResolvedFileName: getResolvedModule,
+                shouldRetryResolution: resolution => !resolution.resolvedModule || !resolutionExtensionIsTSOrJson(resolution.resolvedModule.extension),
+                reusedNames,
+                logChanges: logChangesWhenResolvingModule
+            });
         }
 
         function getResolvedModuleWithFailedLookupLocationsFromCache(moduleName: string, containingFile: string): CachedResolvedModuleWithFailedLookupLocations | undefined {
@@ -498,28 +516,40 @@ namespace ts {
             return fileExtensionIsOneOf(path, failedLookupDefaultExtensions);
         }
 
-        function watchFailedLookupLocationsOfExternalModuleResolutions(name: string, resolution: ResolutionWithFailedLookupLocations) {
-            // No need to set the resolution refCount
-            if (resolution.failedLookupLocations && resolution.failedLookupLocations.length) {
-                if (resolution.refCount) {
-                    resolution.refCount++;
+        function watchFailedLookupLocationsOfExternalModuleResolutions<T extends ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName>(
+            name: string,
+            resolution: T,
+            filePath: Path,
+            getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName<T, R>,
+        ) {
+            if (resolution.refCount) {
+                resolution.refCount++;
+                Debug.assertDefined(resolution.files);
+            }
+            else {
+                resolution.refCount = 1;
+                Debug.assert(resolution.files === undefined);
+                if (isExternalModuleNameRelative(name)) {
+                    watchFailedLookupLocationOfResolution(resolution);
                 }
                 else {
-                    resolution.refCount = 1;
-                    if (isExternalModuleNameRelative(name)) {
-                        watchFailedLookupLocationOfResolution(resolution);
-                    }
-                    else {
-                        nonRelativeExternalModuleResolutions.add(name, resolution);
-                    }
+                    nonRelativeExternalModuleResolutions.add(name, resolution);
+                }
+                const resolved = getResolutionWithResolvedFileName(resolution);
+                if (resolved && resolved.resolvedFileName) {
+                    resolvedFileToResolution.add(resolutionHost.toPath(resolved.resolvedFileName), resolution);
                 }
             }
+            (resolution.files || (resolution.files = [])).push(filePath);
         }
 
         function watchFailedLookupLocationOfResolution(resolution: ResolutionWithFailedLookupLocations) {
             Debug.assert(!!resolution.refCount);
 
             const { failedLookupLocations } = resolution;
+            if (!failedLookupLocations.length) return;
+            resolutionsWithFailedLookups.push(resolution);
+
             let setAtRoot = false;
             for (const failedLookupLocation of failedLookupLocations) {
                 const failedLookupLocationPath = resolutionHost.toPath(failedLookupLocation);
@@ -548,15 +578,11 @@ namespace ts {
             }
         }
 
-        function setRefCountToUndefined(resolution: ResolutionWithFailedLookupLocations) {
-            resolution.refCount = undefined;
-        }
-
         function watchFailedLookupLocationOfNonRelativeModuleResolutions(resolutions: ResolutionWithFailedLookupLocations[], name: string) {
             const program = resolutionHost.getCurrentProgram();
-            const updateResolution = program && program.getTypeChecker().tryFindAmbientModuleWithoutAugmentations(name) ?
-                setRefCountToUndefined : watchFailedLookupLocationOfResolution;
-            resolutions.forEach(updateResolution);
+            if (!program || !program.getTypeChecker().tryFindAmbientModuleWithoutAugmentations(name)) {
+                resolutions.forEach(watchFailedLookupLocationOfResolution);
+            }
         }
 
         function setDirectoryWatcher(dir: string, dirPath: Path, nonRecursive?: boolean) {
@@ -570,13 +596,23 @@ namespace ts {
             }
         }
 
-        function stopWatchFailedLookupLocationOfResolution(resolution: ResolutionWithFailedLookupLocations) {
-            if (!resolution.refCount) {
+        function stopWatchFailedLookupLocationOfResolution<T extends ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName>(
+            resolution: T,
+            filePath: Path,
+            getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName<T, R>,
+        ) {
+            unorderedRemoveItem(Debug.assertDefined(resolution.files), filePath);
+            resolution.refCount!--;
+            if (resolution.refCount) {
                 return;
             }
+            const resolved = getResolutionWithResolvedFileName(resolution);
+            if (resolved && resolved.resolvedFileName) {
+                resolvedFileToResolution.remove(resolutionHost.toPath(resolved.resolvedFileName), resolution);
+            }
 
-            resolution.refCount--;
-            if (resolution.refCount) {
+            if (!unorderedRemoveItem(resolutionsWithFailedLookups, resolution)) {
+                // If not watching failed lookups, it wont be there in resolutionsWithFailedLookups
                 return;
             }
 
@@ -625,17 +661,21 @@ namespace ts {
                     cachedDirectoryStructureHost.addOrDeleteFileOrDirectory(fileOrDirectory, fileOrDirectoryPath);
                 }
 
-                if (!allFilesHaveInvalidatedResolution && invalidateResolutionOfFailedLookupLocation(fileOrDirectoryPath, dirPath === fileOrDirectoryPath)) {
+                if (invalidateResolutionOfFailedLookupLocation(fileOrDirectoryPath, dirPath === fileOrDirectoryPath)) {
                     resolutionHost.onInvalidatedResolution();
                 }
             }, nonRecursive ? WatchDirectoryFlags.None : WatchDirectoryFlags.Recursive);
         }
 
-        function removeResolutionsOfFileFromCache(cache: Map<Map<ResolutionWithFailedLookupLocations>>, filePath: Path) {
+        function removeResolutionsOfFileFromCache<T extends ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName>(
+            cache: Map<Map<T>>,
+            filePath: Path,
+            getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName<T, R>,
+        ) {
             // Deleted file, stop watching failed lookups for all the resolutions in the file
             const resolutions = cache.get(filePath);
             if (resolutions) {
-                resolutions.forEach(stopWatchFailedLookupLocationOfResolution);
+                resolutions.forEach(resolution => stopWatchFailedLookupLocationOfResolution(resolution, filePath, getResolutionWithResolvedFileName));
                 cache.delete(filePath);
             }
         }
@@ -655,69 +695,27 @@ namespace ts {
         }
 
         function removeResolutionsOfFile(filePath: Path) {
-            removeResolutionsOfFileFromCache(resolvedModuleNames, filePath);
-            removeResolutionsOfFileFromCache(resolvedTypeReferenceDirectives, filePath);
-        }
-
-        function invalidateResolutionCache<T extends ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName>(
-            cache: Map<Map<T>>,
-            isInvalidatedResolution: (resolution: T, getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName<T, R>) => boolean,
-            getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName<T, R>
-        ) {
-            const seen = createMap<Map<true>>();
-            cache.forEach((resolutions, containingFilePath) => {
-                const dirPath = getDirectoryPath(containingFilePath);
-                let seenInDir = seen.get(dirPath);
-                if (!seenInDir) {
-                    seenInDir = createMap<true>();
-                    seen.set(dirPath, seenInDir);
-                }
-                resolutions.forEach((resolution, name) => {
-                    if (seenInDir!.has(name)) {
-                        return;
-                    }
-                    seenInDir!.set(name, true);
-                    if (!resolution.isInvalidated && isInvalidatedResolution(resolution, getResolutionWithResolvedFileName)) {
-                        // Mark the file as needing re-evaluation of module resolution instead of using it blindly.
-                        resolution.isInvalidated = true;
-                        (filesWithInvalidatedResolutions || (filesWithInvalidatedResolutions = createMap<true>())).set(containingFilePath, true);
-
-                        // When its a file with inferred types resolution, invalidate type reference directive resolution
-                        if (endsWith(containingFilePath, inferredTypesContainingFile)) {
-                            resolutionHost.onChangedAutomaticTypeDirectiveNames();
-                        }
-                    }
-                });
-            });
+            removeResolutionsOfFileFromCache(resolvedModuleNames, filePath, getResolvedModule);
+            removeResolutionsOfFileFromCache(resolvedTypeReferenceDirectives, filePath, getResolvedTypeReferenceDirective);
         }
 
-        function hasReachedResolutionIterationLimit() {
-            const maxSize = resolutionHost.maxNumberOfFilesToIterateForInvalidation || maxNumberOfFilesToIterateForInvalidation;
-            return resolvedModuleNames.size > maxSize || resolvedTypeReferenceDirectives.size > maxSize;
-        }
-
-        function invalidateResolutions(
-            isInvalidatedResolution: (resolution: ResolutionWithFailedLookupLocations, getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName) => boolean,
-        ) {
-            // If more than maxNumberOfFilesToIterateForInvalidation present,
-            // just invalidated all files and recalculate the resolutions for files instead
-            if (hasReachedResolutionIterationLimit()) {
-                allFilesHaveInvalidatedResolution = true;
-                return;
+        function invalidateResolution(resolution: ResolutionWithFailedLookupLocations) {
+            resolution.isInvalidated = true;
+            let changedInAutoTypeReferenced = false;
+            for (const containingFilePath of Debug.assertDefined(resolution.files)) {
+                (filesWithInvalidatedResolutions || (filesWithInvalidatedResolutions = createMap<true>())).set(containingFilePath, true);
+                // When its a file with inferred types resolution, invalidate type reference directive resolution
+                changedInAutoTypeReferenced = changedInAutoTypeReferenced || containingFilePath.endsWith(inferredTypesContainingFile);
+            }
+            if (changedInAutoTypeReferenced) {
+                resolutionHost.onChangedAutomaticTypeDirectiveNames();
             }
-            invalidateResolutionCache(resolvedModuleNames, isInvalidatedResolution, getResolvedModule);
-            invalidateResolutionCache(resolvedTypeReferenceDirectives, isInvalidatedResolution, getResolvedTypeReferenceDirective);
         }
 
         function invalidateResolutionOfFile(filePath: Path) {
             removeResolutionsOfFile(filePath);
-            invalidateResolutions(
-                // Resolution is invalidated if the resulting file name is same as the deleted file path
-                (resolution, getResolutionWithResolvedFileName) => {
-                    const result = getResolutionWithResolvedFileName(resolution);
-                    return !!result && resolutionHost.toPath(result.resolvedFileName!) === filePath; // TODO: GH#18217
-                }
-            );
+            // Resolution is invalidated if the resulting file name is same as the deleted file path
+            forEach(resolvedFileToResolution.get(filePath), invalidateResolution);
         }
 
         function setFilesWithInvalidatedNonRelativeUnresolvedImports(filesMap: ReadonlyMap<readonly string[]>) {
@@ -766,13 +764,15 @@ namespace ts {
                     isChangedFailedLookupLocation = location => resolutionHost.toPath(location) === fileOrDirectoryPath;
                 }
             }
-            const hasChangedFailedLookupLocation = (resolution: ResolutionWithFailedLookupLocations) => some(resolution.failedLookupLocations, isChangedFailedLookupLocation);
-            const invalidatedFilesCount = filesWithInvalidatedResolutions && filesWithInvalidatedResolutions.size;
-            invalidateResolutions(
-                // Resolution is invalidated if the resulting file name is same as the deleted file path
-                hasChangedFailedLookupLocation
-            );
-            return allFilesHaveInvalidatedResolution || filesWithInvalidatedResolutions && filesWithInvalidatedResolutions.size !== invalidatedFilesCount;
+            let invalidated = false;
+            // Resolution is invalidated if the resulting file name is same as the deleted file path
+            for (const resolution of resolutionsWithFailedLookups) {
+                if (resolution.failedLookupLocations.some(isChangedFailedLookupLocation)) {
+                    invalidateResolution(resolution);
+                    invalidated = true;
+                }
+            }
+            return invalidated;
         }
 
         function closeTypeRootsWatch() {
@@ -780,10 +780,6 @@ namespace ts {
         }
 
         function getDirectoryToWatchFailedLookupLocationFromTypeRoot(typeRoot: string, typeRootPath: Path): Path | undefined {
-            if (allFilesHaveInvalidatedResolution) {
-                return undefined;
-            }
-
             if (isInDirectoryPath(rootPath, typeRootPath)) {
                 return rootPath;
             }
diff --git a/src/compiler/types.ts b/src/compiler/types.ts
index 8348249b77a9f..298586f1dcadb 100644
--- a/src/compiler/types.ts
+++ b/src/compiler/types.ts
@@ -5635,7 +5635,7 @@ namespace ts {
     export interface ResolvedModuleWithFailedLookupLocations {
         readonly resolvedModule: ResolvedModuleFull | undefined;
         /* @internal */
-        readonly failedLookupLocations: readonly string[];
+        readonly failedLookupLocations: string[];
     }
 
     export interface ResolvedTypeReferenceDirective {
@@ -5650,7 +5650,7 @@ namespace ts {
 
     export interface ResolvedTypeReferenceDirectiveWithFailedLookupLocations {
         readonly resolvedTypeReferenceDirective: ResolvedTypeReferenceDirective | undefined;
-        readonly failedLookupLocations: readonly string[];
+        readonly failedLookupLocations: string[];
     }
 
     /* @internal */
diff --git a/src/compiler/watchPublic.ts b/src/compiler/watchPublic.ts
index c5f3928adef9d..0850466f9fd72 100644
--- a/src/compiler/watchPublic.ts
+++ b/src/compiler/watchPublic.ts
@@ -116,10 +116,6 @@ namespace ts {
     export interface WatchCompilerHost<T extends BuilderProgram> extends ProgramHost<T>, WatchHost {
         /** If provided, callback to invoke after every new program creation */
         afterProgramCreate?(program: T): void;
-
-        // Only for testing
-        /*@internal*/
-        maxNumberOfFilesToIterateForInvalidation?: number;
     }
 
     /**
@@ -293,7 +289,6 @@ namespace ts {
             scheduleProgramUpdate();
         };
         compilerHost.fileIsOpen = returnFalse;
-        compilerHost.maxNumberOfFilesToIterateForInvalidation = host.maxNumberOfFilesToIterateForInvalidation;
         compilerHost.getCurrentProgram = getCurrentProgram;
         compilerHost.writeLog = writeLog;
 
diff --git a/src/executeCommandLine/executeCommandLine.ts b/src/executeCommandLine/executeCommandLine.ts
index e66b55d9b558e..7d5b434b8e684 100644
--- a/src/executeCommandLine/executeCommandLine.ts
+++ b/src/executeCommandLine/executeCommandLine.ts
@@ -156,7 +156,6 @@ namespace ts {
         sys: System,
         cb: ExecuteCommandLineCallbacks,
         commandLine: ParsedCommandLine,
-        maxNumberOfFilesToIterateForInvalidation: number | undefined
     ) {
         let reportDiagnostic = createDiagnosticReporter(sys);
         if (commandLine.options.build) {
@@ -271,7 +270,6 @@ namespace ts {
                     configParseResult,
                     commandLineOptions,
                     commandLine.watchOptions,
-                    maxNumberOfFilesToIterateForInvalidation
                 );
             }
             else if (isIncrementalCompilation(configParseResult.options)) {
@@ -311,7 +309,6 @@ namespace ts {
                     commandLine.fileNames,
                     commandLineOptions,
                     commandLine.watchOptions,
-                    maxNumberOfFilesToIterateForInvalidation
                 );
             }
             else if (isIncrementalCompilation(commandLineOptions)) {
@@ -346,7 +343,6 @@ namespace ts {
         system: System,
         cb: ExecuteCommandLineCallbacks,
         commandLineArgs: readonly string[],
-        maxNumberOfFilesToIterateForInvalidation?: number
     ) {
         if (isBuild(commandLineArgs)) {
             const { buildOptions, watchOptions, projects, errors } = parseBuildCommand(commandLineArgs.slice(1));
@@ -378,11 +374,10 @@ namespace ts {
                 system,
                 cb,
                 commandLine,
-                maxNumberOfFilesToIterateForInvalidation
             ));
         }
         else {
-            return executeCommandLineWorker(system, cb, commandLine, maxNumberOfFilesToIterateForInvalidation);
+            return executeCommandLineWorker(system, cb, commandLine);
         }
     }
 
@@ -556,10 +551,8 @@ namespace ts {
         sys: System,
         cb: ExecuteCommandLineCallbacks,
         watchCompilerHost: WatchCompilerHost<EmitAndSemanticDiagnosticsBuilderProgram>,
-        maxNumberOfFilesToIterateForInvalidation: number | undefined
     ) {
         updateCreateProgram(sys, watchCompilerHost);
-        watchCompilerHost.maxNumberOfFilesToIterateForInvalidation = maxNumberOfFilesToIterateForInvalidation;
         const emitFilesUsingBuilder = watchCompilerHost.afterProgramCreate!; // TODO: GH#18217
         watchCompilerHost.afterProgramCreate = builderProgram => {
             emitFilesUsingBuilder(builderProgram);
@@ -579,7 +572,6 @@ namespace ts {
         configParseResult: ParsedCommandLine,
         optionsToExtend: CompilerOptions,
         watchOptionsToExtend: WatchOptions | undefined,
-        maxNumberOfFilesToIterateForInvalidation: number | undefined
     ) {
         const watchCompilerHost = createWatchCompilerHostOfConfigFile(
             configParseResult.options.configFilePath!,
@@ -590,7 +582,7 @@ namespace ts {
             reportDiagnostic,
             createWatchStatusReporter(sys, configParseResult.options)
         ); // TODO: GH#18217
-        updateWatchCompilationHost(sys, cb, watchCompilerHost, maxNumberOfFilesToIterateForInvalidation);
+        updateWatchCompilationHost(sys, cb, watchCompilerHost);
         watchCompilerHost.configFileParsingResult = configParseResult;
         return createWatchProgram(watchCompilerHost);
     }
@@ -602,7 +594,6 @@ namespace ts {
         rootFiles: string[],
         options: CompilerOptions,
         watchOptions: WatchOptions | undefined,
-        maxNumberOfFilesToIterateForInvalidation: number | undefined
     ) {
         const watchCompilerHost = createWatchCompilerHostOfFilesAndCompilerOptions(
             rootFiles,
@@ -613,7 +604,7 @@ namespace ts {
             reportDiagnostic,
             createWatchStatusReporter(sys, options)
         );
-        updateWatchCompilationHost(sys, cb, watchCompilerHost, maxNumberOfFilesToIterateForInvalidation);
+        updateWatchCompilationHost(sys, cb, watchCompilerHost);
         return createWatchProgram(watchCompilerHost);
     }
 
diff --git a/src/testRunner/unittests/tscWatch/helpers.ts b/src/testRunner/unittests/tscWatch/helpers.ts
index 01d9d1c42621d..5492b5cbfe63b 100644
--- a/src/testRunner/unittests/tscWatch/helpers.ts
+++ b/src/testRunner/unittests/tscWatch/helpers.ts
@@ -33,15 +33,13 @@ namespace ts.tscWatch {
 
     export type Watch = WatchOfConfigFile<EmitAndSemanticDiagnosticsBuilderProgram> | WatchOfFilesAndCompilerOptions<EmitAndSemanticDiagnosticsBuilderProgram>;
 
-    export function createWatchOfConfigFile(configFileName: string, host: WatchedSystem, optionsToExtend?: CompilerOptions, watchOptionsToExtend?: WatchOptions, maxNumberOfFilesToIterateForInvalidation?: number) {
+    export function createWatchOfConfigFile(configFileName: string, host: WatchedSystem, optionsToExtend?: CompilerOptions, watchOptionsToExtend?: WatchOptions) {
         const compilerHost = createWatchCompilerHostOfConfigFile(configFileName, optionsToExtend || {}, watchOptionsToExtend, host);
-        compilerHost.maxNumberOfFilesToIterateForInvalidation = maxNumberOfFilesToIterateForInvalidation;
         return createWatchProgram(compilerHost);
     }
 
-    export function createWatchOfFilesAndCompilerOptions(rootFiles: string[], host: WatchedSystem, options: CompilerOptions = {}, watchOptions?: WatchOptions, maxNumberOfFilesToIterateForInvalidation?: number) {
+    export function createWatchOfFilesAndCompilerOptions(rootFiles: string[], host: WatchedSystem, options: CompilerOptions = {}, watchOptions?: WatchOptions) {
         const compilerHost = createWatchCompilerHostOfFilesAndCompilerOptions(rootFiles, options, watchOptions, host);
-        compilerHost.maxNumberOfFilesToIterateForInvalidation = maxNumberOfFilesToIterateForInvalidation;
         return createWatchProgram(compilerHost);
     }
 
@@ -288,7 +286,6 @@ namespace ts.tscWatch {
     }
     export interface TscWatchCompile extends TscWatchCompileBase {
         sys: () => WatchedSystem;
-        maxNumberOfFilesToIterateForInvalidation?: number;
     }
 
     export type SystemSnap = ReturnType<WatchedSystem["snap"]>;
@@ -308,7 +305,6 @@ namespace ts.tscWatch {
                 sys,
                 cb,
                 commandLineArgs,
-                input.maxNumberOfFilesToIterateForInvalidation
             );
             runWatchBaseline({
                 scenario,
diff --git a/src/testRunner/unittests/tscWatch/programUpdates.ts b/src/testRunner/unittests/tscWatch/programUpdates.ts
index 25c820a9f51a1..7e6660805e043 100644
--- a/src/testRunner/unittests/tscWatch/programUpdates.ts
+++ b/src/testRunner/unittests/tscWatch/programUpdates.ts
@@ -984,7 +984,6 @@ declare const eval: any`
                     scenario,
                     subScenario: `should not trigger recompilation because of program emit/${subScenario}`,
                     commandLineArgs: ["-w", "-p", `${projectRoot}/tsconfig.json`],
-                    maxNumberOfFilesToIterateForInvalidation: 1,
                     sys: () => {
                         const file1: File = {
                             path: `${projectRoot}/file1.ts`,
diff --git a/src/testRunner/unittests/tscWatch/resolutionCache.ts b/src/testRunner/unittests/tscWatch/resolutionCache.ts
index cb2c17cf065f2..df06ed11a41c7 100644
--- a/src/testRunner/unittests/tscWatch/resolutionCache.ts
+++ b/src/testRunner/unittests/tscWatch/resolutionCache.ts
@@ -320,7 +320,6 @@ declare module "fs" {
                     scenario,
                     subScenario: `ignores changes in node_modules that start with dot/${subScenario}`,
                     commandLineArgs,
-                    maxNumberOfFilesToIterateForInvalidation: 1,
                     sys: () => {
                         const file1: File = {
                             path: `${projectRoot}/test.ts`,
diff --git a/src/testRunner/unittests/tsserver/events/projectUpdatedInBackground.ts b/src/testRunner/unittests/tsserver/events/projectUpdatedInBackground.ts
index 958202cdd19a8..c247e2c224ddf 100644
--- a/src/testRunner/unittests/tsserver/events/projectUpdatedInBackground.ts
+++ b/src/testRunner/unittests/tsserver/events/projectUpdatedInBackground.ts
@@ -418,7 +418,7 @@ namespace ts.projectSystem {
             });
 
             describe("resolution when resolution cache size", () => {
-                function verifyWithMaxCacheLimit(limitHit: boolean, useSlashRootAsSomeNotRootFolderInUserDirectory: boolean) {
+                function verifyWithMaxCacheLimit(useSlashRootAsSomeNotRootFolderInUserDirectory: boolean) {
                     const rootFolder = useSlashRootAsSomeNotRootFolderInUserDirectory ? "/user/username/rootfolder/otherfolder/" : "/";
                     const file1: File = {
                         path: rootFolder + "a/b/project/file1.ts",
@@ -451,9 +451,6 @@ namespace ts.projectSystem {
                     checkNumberOfProjects(projectService, { configuredProjects: 1 });
                     const project = projectService.configuredProjects.get(configFile.path)!;
                     verifyProject();
-                    if (limitHit) {
-                        (project as ResolutionCacheHost).maxNumberOfFilesToIterateForInvalidation = 1;
-                    }
 
                     file3.content += "export class d {}";
                     host.reloadFS(projectFiles);
@@ -495,20 +492,12 @@ namespace ts.projectSystem {
                     }
                 }
 
-                it("limit not hit and project is not at root level", () => {
-                    verifyWithMaxCacheLimit(/*limitHit*/ false, /*useSlashRootAsSomeNotRootFolderInUserDirectory*/ true);
-                });
-
-                it("limit hit and project is not at root level", () => {
-                    verifyWithMaxCacheLimit(/*limitHit*/ true, /*useSlashRootAsSomeNotRootFolderInUserDirectory*/ true);
-                });
-
-                it("limit not hit and project is at root level", () => {
-                    verifyWithMaxCacheLimit(/*limitHit*/ false, /*useSlashRootAsSomeNotRootFolderInUserDirectory*/ false);
+                it("project is not at root level", () => {
+                    verifyWithMaxCacheLimit(/*useSlashRootAsSomeNotRootFolderInUserDirectory*/ true);
                 });
 
-                it("limit hit and project is at root level", () => {
-                    verifyWithMaxCacheLimit(/*limitHit*/ true, /*useSlashRootAsSomeNotRootFolderInUserDirectory*/ false);
+                it("project is at root level", () => {
+                    verifyWithMaxCacheLimit(/*useSlashRootAsSomeNotRootFolderInUserDirectory*/ false);
                 });
             });
         }
diff --git a/src/testRunner/unittests/tsserver/resolutionCache.ts b/src/testRunner/unittests/tsserver/resolutionCache.ts
index ed6de15a16886..c1f4c755a3a59 100644
--- a/src/testRunner/unittests/tsserver/resolutionCache.ts
+++ b/src/testRunner/unittests/tsserver/resolutionCache.ts
@@ -902,7 +902,6 @@ export const x = 10;`
                 checkNumberOfProjects(service, { inferredProjects: 1 });
                 const project = service.inferredProjects[0];
                 checkProjectActualFiles(project, files.map(f => f.path));
-                (project as ResolutionCacheHost).maxNumberOfFilesToIterateForInvalidation = 1;
                 host.checkTimeoutQueueLength(0);
 
                 host.ensureFileOrFolder(npmCacheFile);
@@ -943,8 +942,6 @@ export const x = 10;`
                 const resolutionTrace = createHostModuleResolutionTrace(host);
                 const service = createProjectService(host);
                 service.openClientFile(file1.path);
-                const project = service.configuredProjects.get(configFile.path)!;
-                (project as ResolutionCacheHost).maxNumberOfFilesToIterateForInvalidation = 1;
                 const expectedTrace = getExpectedNonRelativeModuleResolutionTrace(host, file1, module1, module1Name);
                 getExpectedNonRelativeModuleResolutionTrace(host, file1, module2, module2Name, expectedTrace);
                 verifyTrace(resolutionTrace, expectedTrace);
diff --git a/tests/baselines/reference/api/tsserverlibrary.d.ts b/tests/baselines/reference/api/tsserverlibrary.d.ts
index 2cdbe454e68d2..6488461fc28e3 100644
--- a/tests/baselines/reference/api/tsserverlibrary.d.ts
+++ b/tests/baselines/reference/api/tsserverlibrary.d.ts
@@ -2918,7 +2918,7 @@ declare namespace ts {
     }
     export interface ResolvedTypeReferenceDirectiveWithFailedLookupLocations {
         readonly resolvedTypeReferenceDirective: ResolvedTypeReferenceDirective | undefined;
-        readonly failedLookupLocations: readonly string[];
+        readonly failedLookupLocations: string[];
     }
     export interface CompilerHost extends ModuleResolutionHost {
         getSourceFile(fileName: string, languageVersion: ScriptTarget, onError?: (message: string) => void, shouldCreateNewSourceFile?: boolean): SourceFile | undefined;
diff --git a/tests/baselines/reference/api/typescript.d.ts b/tests/baselines/reference/api/typescript.d.ts
index 844fe42758fef..c3f629b355c8f 100644
--- a/tests/baselines/reference/api/typescript.d.ts
+++ b/tests/baselines/reference/api/typescript.d.ts
@@ -2918,7 +2918,7 @@ declare namespace ts {
     }
     export interface ResolvedTypeReferenceDirectiveWithFailedLookupLocations {
         readonly resolvedTypeReferenceDirective: ResolvedTypeReferenceDirective | undefined;
-        readonly failedLookupLocations: readonly string[];
+        readonly failedLookupLocations: string[];
     }
     export interface CompilerHost extends ModuleResolutionHost {
         getSourceFile(fileName: string, languageVersion: ScriptTarget, onError?: (message: string) => void, shouldCreateNewSourceFile?: boolean): SourceFile | undefined;