diff --git a/src/Workspaces/Core/Portable/Workspace/Solution/CachingAnalyzerConfigSet.cs b/src/Workspaces/Core/Portable/Workspace/Solution/CachingAnalyzerConfigSet.cs
new file mode 100644
index 0000000000000..70e05cad20b9d
--- /dev/null
+++ b/src/Workspaces/Core/Portable/Workspace/Solution/CachingAnalyzerConfigSet.cs
@@ -0,0 +1,29 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#nullable enable
+
+using System;
+using System.Collections.Concurrent;
+
+namespace Microsoft.CodeAnalysis
+{
+    internal sealed class CachingAnalyzerConfigSet
+    {
+        private readonly ConcurrentDictionary<string, AnalyzerConfigOptionsResult> _sourcePathToResult = new ConcurrentDictionary<string, AnalyzerConfigOptionsResult>();
+        private readonly Func<string, AnalyzerConfigOptionsResult> _computeFunction;
+        private readonly AnalyzerConfigSet _underlyingSet;
+
+        public CachingAnalyzerConfigSet(AnalyzerConfigSet underlyingSet)
+        {
+            _underlyingSet = underlyingSet;
+            _computeFunction = _underlyingSet.GetOptionsForSourcePath;
+        }
+
+        public AnalyzerConfigOptionsResult GetOptionsForSourcePath(string sourcePath)
+        {
+            return _sourcePathToResult.GetOrAdd(sourcePath, _computeFunction);
+        }
+    }
+}
diff --git a/src/Workspaces/Core/Portable/Workspace/Solution/ProjectState.cs b/src/Workspaces/Core/Portable/Workspace/Solution/ProjectState.cs
index 07612bc422971..86e59f11e2251 100644
--- a/src/Workspaces/Core/Portable/Workspace/Solution/ProjectState.cs
+++ b/src/Workspaces/Core/Portable/Workspace/Solution/ProjectState.cs
@@ -5,6 +5,7 @@
 #nullable enable
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Diagnostics;
@@ -55,7 +56,7 @@ internal partial class ProjectState
         /// <summary>
         /// The <see cref="AnalyzerConfigSet"/> to be used for analyzer options for specific trees.
         /// </summary>
-        private readonly ValueSource<AnalyzerConfigSet> _lazyAnalyzerConfigSet;
+        private readonly ValueSource<CachingAnalyzerConfigSet> _lazyAnalyzerConfigSet;
 
         private AnalyzerOptions? _lazyAnalyzerOptions;
 
@@ -70,7 +71,7 @@ private ProjectState(
             ImmutableSortedDictionary<DocumentId, AnalyzerConfigDocumentState> analyzerConfigDocumentStates,
             AsyncLazy<VersionStamp> lazyLatestDocumentVersion,
             AsyncLazy<VersionStamp> lazyLatestDocumentTopLevelChangeVersion,
-            ValueSource<AnalyzerConfigSet> lazyAnalyzerConfigSet)
+            ValueSource<CachingAnalyzerConfigSet> lazyAnalyzerConfigSet)
         {
             _solutionServices = solutionServices;
             _languageServices = languageServices;
@@ -321,6 +322,7 @@ public override AnalyzerConfigOptions GetOptions(AdditionalText textFile)
                 // TODO: correctly find the file path, since it looks like we give this the document's .Name under the covers if we don't have one
                 return new WorkspaceAnalyzerConfigOptions(_projectState._lazyAnalyzerConfigSet.GetValue(CancellationToken.None).GetOptionsForSourcePath(textFile.Path));
             }
+
             private sealed class WorkspaceAnalyzerConfigOptions : AnalyzerConfigOptions
             {
                 private readonly ImmutableDictionary<string, string> _backing;
@@ -334,9 +336,9 @@ public WorkspaceAnalyzerConfigOptions(AnalyzerConfigOptionsResult analyzerConfig
 
         private sealed class WorkspaceSyntaxTreeOptionsProvider : SyntaxTreeOptionsProvider
         {
-            private readonly ValueSource<AnalyzerConfigSet> _lazyAnalyzerConfigSet;
+            private readonly ValueSource<CachingAnalyzerConfigSet> _lazyAnalyzerConfigSet;
 
-            public WorkspaceSyntaxTreeOptionsProvider(ValueSource<AnalyzerConfigSet> lazyAnalyzerConfigSet)
+            public WorkspaceSyntaxTreeOptionsProvider(ValueSource<CachingAnalyzerConfigSet> lazyAnalyzerConfigSet)
                 => _lazyAnalyzerConfigSet = lazyAnalyzerConfigSet;
 
             public override bool? IsGenerated(SyntaxTree tree)
@@ -362,9 +364,9 @@ public override bool Equals(object? obj)
             public override int GetHashCode() => _lazyAnalyzerConfigSet.GetHashCode();
         }
 
-        private static ValueSource<AnalyzerConfigSet> ComputeAnalyzerConfigSetValueSource(IEnumerable<AnalyzerConfigDocumentState> analyzerConfigDocumentStates)
+        private static ValueSource<CachingAnalyzerConfigSet> ComputeAnalyzerConfigSetValueSource(IEnumerable<AnalyzerConfigDocumentState> analyzerConfigDocumentStates)
         {
-            return new AsyncLazy<AnalyzerConfigSet>(
+            return new AsyncLazy<CachingAnalyzerConfigSet>(
                 asynchronousComputeFunction: async cancellationToken =>
                 {
                     var tasks = analyzerConfigDocumentStates.Select(a => a.GetAnalyzerConfigAsync(cancellationToken));
@@ -372,12 +374,12 @@ private static ValueSource<AnalyzerConfigSet> ComputeAnalyzerConfigSetValueSourc
 
                     cancellationToken.ThrowIfCancellationRequested();
 
-                    return AnalyzerConfigSet.Create(analyzerConfigs);
+                    return new CachingAnalyzerConfigSet(AnalyzerConfigSet.Create(analyzerConfigs));
                 },
                 synchronousComputeFunction: cancellationToken =>
                 {
                     var analyzerConfigs = analyzerConfigDocumentStates.SelectAsArray(a => a.GetAnalyzerConfig(cancellationToken));
-                    return AnalyzerConfigSet.Create(analyzerConfigs);
+                    return new CachingAnalyzerConfigSet(AnalyzerConfigSet.Create(analyzerConfigs));
                 },
                 cacheResult: true);
         }
@@ -522,7 +524,7 @@ private ProjectState With(
             ImmutableSortedDictionary<DocumentId, AnalyzerConfigDocumentState>? analyzerConfigDocumentStates = null,
             AsyncLazy<VersionStamp>? latestDocumentVersion = null,
             AsyncLazy<VersionStamp>? latestDocumentTopLevelChangeVersion = null,
-            ValueSource<AnalyzerConfigSet>? analyzerConfigSet = null)
+            ValueSource<CachingAnalyzerConfigSet>? analyzerConfigSet = null)
         {
             return new ProjectState(
                 projectInfo ?? _projectInfo,
@@ -695,6 +697,7 @@ private ProjectState CreateNewStateForChangedAnalyzerConfigDocuments(ImmutableSo
                 projectInfo = projectInfo
                     .WithCompilationOptions(CompilationOptions.WithSyntaxTreeOptionsProvider(newProvider));
             }
+
             return this.With(
                 projectInfo: projectInfo,
                 analyzerConfigDocumentStates: newAnalyzerConfigDocumentStates,
@@ -703,10 +706,13 @@ private ProjectState CreateNewStateForChangedAnalyzerConfigDocuments(ImmutableSo
 
         public ProjectState RemoveDocuments(ImmutableArray<DocumentId> documentIds)
         {
+            // We create a new CachingAnalyzerConfigSet for the new snapshot to avoid holding onto cached information
+            // for removed documents.
             return this.With(
                 projectInfo: this.ProjectInfo.WithVersion(this.Version.GetNewerVersion()),
                 documentIds: _documentIds.RemoveRange(documentIds),
-                documentStates: _documentStates.RemoveRange(documentIds));
+                documentStates: _documentStates.RemoveRange(documentIds),
+                analyzerConfigSet: ComputeAnalyzerConfigSetValueSource(AnalyzerConfigDocumentStates.Values));
         }
 
         public ProjectState RemoveAdditionalDocuments(ImmutableArray<DocumentId> documentIds)
@@ -726,10 +732,13 @@ public ProjectState RemoveAnalyzerConfigDocuments(ImmutableArray<DocumentId> doc
 
         public ProjectState RemoveAllDocuments()
         {
+            // We create a new CachingAnalyzerConfigSet for the new snapshot to avoid holding onto cached information
+            // for removed documents.
             return this.With(
                 projectInfo: this.ProjectInfo.WithVersion(this.Version.GetNewerVersion()).WithDocuments(SpecializedCollections.EmptyEnumerable<DocumentInfo>()),
                 documentIds: ImmutableList<DocumentId>.Empty,
-                documentStates: ImmutableSortedDictionary.Create<DocumentId, DocumentState>(DocumentIdComparer.Instance));
+                documentStates: ImmutableSortedDictionary.Create<DocumentId, DocumentState>(DocumentIdComparer.Instance),
+                analyzerConfigSet: ComputeAnalyzerConfigSetValueSource(AnalyzerConfigDocumentStates.Values));
         }
 
         public ProjectState UpdateDocument(DocumentState newDocument, bool textChanged, bool recalculateDependentVersions)