Sync project level info in parallel.

src/VisualStudio/Core/Test.Next/Services/AssetProviderTests.cs

@@ -12,6 +12,7 @@
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.Editor.UnitTests.Workspaces;
+using Microsoft.CodeAnalysis.PooledObjects;
 using Microsoft.CodeAnalysis.Remote;
 using Microsoft.CodeAnalysis.Remote.Testing;
 using Microsoft.CodeAnalysis.Serialization;
@@ -137,7 +138,11 @@ public async Task TestProjectSynchronization()
             var assetSource = new SimpleAssetSource(workspace.Services.GetService<ISerializerService>(), map);
 
             var service = new AssetProvider(sessionId, storage, assetSource, remoteWorkspace.Services.GetService<ISerializerService>());
-            await service.SynchronizeProjectAssetsAsync(await project.State.GetStateChecksumsAsync(CancellationToken.None), CancellationToken.None);
+
+            using var _ = ArrayBuilder<ProjectStateChecksums>.GetInstance(out var allProjectChecksums);
+            allProjectChecksums.Add(await project.State.GetStateChecksumsAsync(CancellationToken.None));
+
+            await service.SynchronizeProjectAssetsAsync(allProjectChecksums, CancellationToken.None);
 
             TestUtils.VerifyAssetStorage(map, storage);
         }

src/Workspaces/Core/Portable/Workspace/Solution/AssetHint.cs

@@ -27,6 +27,8 @@ internal readonly struct AssetPath
     /// </summary>
     public static readonly AssetPath SolutionAndTopLevelProjectsOnly = new(AssetPathKind.Solution | AssetPathKind.TopLevelProjects);
 
+    public static readonly AssetPath ProjectOnly = new(AssetPathKind.Projects);
+
     /// <summary>
     /// Special instance, allowed only in tests/debug-asserts, that can do a full lookup across the entire checksum
     /// tree.  Should not be used in normal release-mode product code.

src/Workspaces/Core/Portable/Workspace/Solution/StateChecksums.cs

@@ -272,14 +272,13 @@ public async Task FindAsync(
 
                 if (projectState.TryGetStateChecksums(out var projectStateChecksums))
                 {
-                    if (searchingChecksumsLeft.Remove(projectStateChecksums.Checksum))
-                        result[projectStateChecksums.Checksum] = projectStateChecksums;
-
-                    if (searchingChecksumsLeft.Remove(projectStateChecksums.Info))
-                        result[projectStateChecksums.Info] = projectState.Attributes;
-
-                    if (searchingChecksumsLeft.Remove(projectStateChecksums.CompilationOptions))
-                        result[projectStateChecksums.CompilationOptions] = projectState.CompilationOptions!;
+                    await projectStateChecksums.FindAsync(
+                        projectState,
+                        // Only search the project level info of the project we're diving into.  Do not go into documents.
+                        AssetPath.ProjectOnly,
+                        searchingChecksumsLeft,
+                        result,
+                        cancellationToken).ConfigureAwait(false);
                 }
             }
         }

src/Workspaces/Remote/ServiceHub/Host/AssetProvider.cs

@@ -8,6 +8,7 @@
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.CodeAnalysis.Diagnostics;
 using Microsoft.CodeAnalysis.Internal.Log;
 using Microsoft.CodeAnalysis.PooledObjects;
 using Microsoft.CodeAnalysis.Serialization;
@@ -22,7 +23,7 @@ namespace Microsoft.CodeAnalysis.Remote;
 internal sealed partial class AssetProvider(Checksum solutionChecksum, SolutionAssetCache assetCache, IAssetSource assetSource, ISerializerService serializerService)
     : AbstractAssetProvider
 {
-    private const int PooledChecksumArraySize = 256;
+    private const int PooledChecksumArraySize = 1024;
     private static readonly ObjectPool<Checksum[]> s_checksumPool = new(() => new Checksum[PooledChecksumArraySize], 16);
 
     private readonly Checksum _solutionChecksum = solutionChecksum;
@@ -109,16 +110,21 @@ await this.SynchronizeAssetsAsync<object, Dictionary<Checksum, object>>(
                 static (checksum, asset, checksumToObjects) => checksumToObjects.Add(checksum, asset),
                 arg: checksumToObjects, cancellationToken).ConfigureAwait(false);
 
+            using var _3 = ArrayBuilder<ProjectStateChecksums>.GetInstance(out var allProjectStateChecksums);
+
             // fourth, get all projects and documents in the solution 
             foreach (var (projectChecksum, _) in stateChecksums.Projects)
             {
                 var projectStateChecksums = (ProjectStateChecksums)checksumToObjects[projectChecksum];
-                await SynchronizeProjectAssetsAsync(projectStateChecksums, cancellationToken).ConfigureAwait(false);
+                allProjectStateChecksums.Add(projectStateChecksums);
             }
+
+            await SynchronizeProjectAssetsAsync(allProjectStateChecksums, cancellationToken).ConfigureAwait(false);
         }
     }
 
-    public async ValueTask SynchronizeProjectAssetsAsync(ProjectStateChecksums projectChecksums, CancellationToken cancellationToken)
+    public async ValueTask SynchronizeProjectAssetsAsync(
+        ArrayBuilder<ProjectStateChecksums> allProjectChecksums, CancellationToken cancellationToken)
     {
         // this will pull in assets that belong to the given project checksum to this remote host. this one is not
         // supposed to be used for functionality but only for perf. that is why it doesn't return anything. to get
@@ -129,58 +135,87 @@ public async ValueTask SynchronizeProjectAssetsAsync(ProjectStateChecksums proje
         // GetAssetAsync call will most likely cache hit. it is most likely since we might change cache heuristic in
         // future which make data to live a lot shorter in the cache, and the data might get expired before one actually
         // consume the data. 
-        using (Logger.LogBlock(FunctionId.AssetService_SynchronizeProjectAssetsAsync, Checksum.GetProjectChecksumsLogInfo, projectChecksums, cancellationToken))
+        using (Logger.LogBlock(FunctionId.AssetService_SynchronizeProjectAssetsAsync, message: null, cancellationToken))
         {
             await SynchronizeProjectAssetsWorkerAsync().ConfigureAwait(false);
         }
 
         async ValueTask SynchronizeProjectAssetsWorkerAsync()
         {
-            // get children of project checksum objects at once
+            using var _ = ArrayBuilder<Task>.GetInstance(out var tasks);
+
+            // Make parallel requests for all the project data across all projects at once.
+            tasks.Add(SynchronizeProjectAssetAsync<ProjectInfo.ProjectAttributes>(static p => p.Info, cancellationToken));
+            tasks.Add(SynchronizeProjectAssetAsync<CompilationOptions>(static p => p.CompilationOptions, cancellationToken));
+            tasks.Add(SynchronizeProjectAssetAsync<ParseOptions>(static p => p.ParseOptions, cancellationToken));
+            tasks.Add(SynchronizeProjectAssetCollectionAsync<ProjectReference>(static p => p.ProjectReferences, cancellationToken));
+            tasks.Add(SynchronizeProjectAssetCollectionAsync<MetadataReference>(static p => p.MetadataReferences, cancellationToken));
+            tasks.Add(SynchronizeProjectAssetCollectionAsync<AnalyzerReference>(static p => p.AnalyzerReferences, cancellationToken));
+
+            // Then sync each project's documents in parallel with each other.
+            foreach (var projectChecksums in allProjectChecksums)
+                tasks.Add(SynchronizeProjectDocumentsAsync(projectChecksums, cancellationToken));
+
+            await Task.WhenAll(tasks).ConfigureAwait(false);
+        }
+
+        static void AddAll(HashSet<Checksum> checksums, ChecksumCollection checksumCollection)
+        {
+            foreach (var checksum in checksumCollection)
+                checksums.Add(checksum);
+        }
+
+        async Task SynchronizeProjectAssetAsync<TAsset>(Func<ProjectStateChecksums, Checksum> getChecksum, CancellationToken cancellationToken)
+        {
+            await Task.Yield();
             using var _ = PooledHashSet<Checksum>.GetInstance(out var checksums);
 
-            checksums.Add(projectChecksums.Info);
-            checksums.Add(projectChecksums.CompilationOptions);
-            checksums.Add(projectChecksums.ParseOptions);
-            AddAll(checksums, projectChecksums.ProjectReferences);
-            AddAll(checksums, projectChecksums.MetadataReferences);
-            AddAll(checksums, projectChecksums.AnalyzerReferences);
-            AddAll(checksums, projectChecksums.Documents.Checksums);
-            AddAll(checksums, projectChecksums.AdditionalDocuments.Checksums);
-            AddAll(checksums, projectChecksums.AnalyzerConfigDocuments.Checksums);
+            foreach (var projectChecksums in allProjectChecksums)
+                checksums.Add(getChecksum(projectChecksums));
 
-            // First synchronize all the top-level info about this project.
-            await this.SynchronizeAssetsAsync<object, VoidResult>(
-                assetPath: AssetPath.ProjectAndDocuments(projectChecksums.ProjectId), checksums, callback: null, arg: default, cancellationToken).ConfigureAwait(false);
+            await this.SynchronizeAssetsAsync<TAsset, VoidResult>(
+                AssetPath.SolutionAndTopLevelProjectsOnly, checksums, callback: null, arg: default, cancellationToken).ConfigureAwait(false);
+        }
 
-            checksums.Clear();
+        async Task SynchronizeProjectAssetCollectionAsync<TAsset>(Func<ProjectStateChecksums, ChecksumCollection> getChecksums, CancellationToken cancellationToken)
+        {
+            await Task.Yield();
+            using var _ = PooledHashSet<Checksum>.GetInstance(out var checksums);
 
-            // Then synchronize the info about all the documents within.
-            await CollectChecksumChildrenAsync(checksums, projectChecksums.Documents).ConfigureAwait(false);
-            await CollectChecksumChildrenAsync(checksums, projectChecksums.AdditionalDocuments).ConfigureAwait(false);
-            await CollectChecksumChildrenAsync(checksums, projectChecksums.AnalyzerConfigDocuments).ConfigureAwait(false);
+            foreach (var projectChecksums in allProjectChecksums)
+                AddAll(checksums, getChecksums(projectChecksums));
 
-            await this.SynchronizeAssetsAsync<object, VoidResult>(
-                assetPath: AssetPath.ProjectAndDocuments(projectChecksums.ProjectId), checksums, callback: null, arg: default, cancellationToken).ConfigureAwait(false);
+            await this.SynchronizeAssetsAsync<TAsset, VoidResult>(
+                AssetPath.SolutionAndTopLevelProjectsOnly, checksums, callback: null, arg: default, cancellationToken).ConfigureAwait(false);
         }
 
-        async ValueTask CollectChecksumChildrenAsync(HashSet<Checksum> checksums, ChecksumsAndIds<DocumentId> collection)
+        async Task SynchronizeProjectDocumentsAsync(ProjectStateChecksums projectChecksums, CancellationToken cancellationToken)
         {
-            // This GetAssetsAsync call should be fast since they were just retrieved above.  There's a small chance
-            // the asset-cache GC pass may have cleaned them up, but that should be exceedingly rare.
-            var allDocChecksums = await this.GetAssetsAsync<DocumentStateChecksums>(
-                AssetPath.ProjectAndDocuments(projectChecksums.ProjectId), collection.Checksums, cancellationToken).ConfigureAwait(false);
-            foreach (var docChecksums in allDocChecksums)
+            await Task.Yield();
+            using var _1 = PooledHashSet<Checksum>.GetInstance(out var checksums);
+
+            AddAll(checksums, projectChecksums.Documents.Checksums);
+            AddAll(checksums, projectChecksums.AdditionalDocuments.Checksums);
+            AddAll(checksums, projectChecksums.AnalyzerConfigDocuments.Checksums);
+
+            // First, fetch all the DocumentStateChecksums for all the documents in the project.
+            using var _2 = ArrayBuilder<DocumentStateChecksums>.GetInstance(out var allDocumentStateChecksums);
+            await this.SynchronizeAssetsAsync<DocumentStateChecksums, ArrayBuilder<DocumentStateChecksums>>(
+                assetPath: AssetPath.ProjectAndDocuments(projectChecksums.ProjectId), checksums,
+                static (_, documentStateChecksums, allDocumentStateChecksums) => allDocumentStateChecksums.Add(documentStateChecksums),
+                allDocumentStateChecksums,
+                cancellationToken).ConfigureAwait(false);
+
+            // Now go and fetch the info and text for all of those documents.
+            checksums.Clear();
+            foreach (var docChecksums in allDocumentStateChecksums)
             {
                 checksums.Add(docChecksums.Info);
                 checksums.Add(docChecksums.Text);
             }
-        }
 
-        static void AddAll(HashSet<Checksum> checksums, ChecksumCollection checksumCollection)
-        {
-            foreach (var checksum in checksumCollection)
-                checksums.Add(checksum);
+            await this.SynchronizeAssetsAsync<object, VoidResult>(
+                assetPath: AssetPath.ProjectAndDocuments(projectChecksums.ProjectId), checksums, callback: null, arg: default, cancellationToken).ConfigureAwait(false);
         }
     }
 

src/Workspaces/Remote/ServiceHub/Host/RemoteWorkspace.SolutionCreator.cs

@@ -264,16 +264,25 @@ await _assetProvider.GetAssetsAsync<CompilationOptions, VoidResult>(
                 }
 
                 using var _2 = ArrayBuilder<ProjectInfo>.GetInstance(out var projectInfos);
+                using var _3 = ArrayBuilder<ProjectStateChecksums>.GetInstance(out var projectStateChecksumsToAdd);
 
                 // added project
                 foreach (var (projectId, newProjectChecksums) in newProjectIdToStateChecksums)
                 {
                     if (!oldProjectIdToStateChecksums.ContainsKey(projectId))
-                    {
-                        // bulk sync added project assets fully since we'll definitely need that data, and we won't want
-                        // to make tons of intermediary calls for it.
+                        projectStateChecksumsToAdd.Add(newProjectChecksums);
+                }
+
+                // bulk sync added project assets fully since we'll definitely need that data, and we won't want
+
+                await _assetProvider.SynchronizeProjectAssetsAsync(projectStateChecksumsToAdd, cancellationToken).ConfigureAwait(false);
 
-                        await _assetProvider.SynchronizeProjectAssetsAsync(newProjectChecksums, cancellationToken).ConfigureAwait(false);
+                foreach (var (projectId, newProjectChecksums) in newProjectIdToStateChecksums)
+                {
+                    if (!oldProjectIdToStateChecksums.ContainsKey(projectId))
+                    {
+                        // Now make a ProjectInfo corresponding to the new project checksums.  This should be fast due
+                        // to the bulk sync we just performed above.
                         var projectInfo = await _assetProvider.CreateProjectInfoAsync(projectId, newProjectChecksums.Checksum, cancellationToken).ConfigureAwait(false);
                         projectInfos.Add(projectInfo);
                     }
@@ -295,7 +304,7 @@ await _assetProvider.GetAssetsAsync<CompilationOptions, VoidResult>(
                     }
                 }
 
-                using var _3 = ArrayBuilder<ProjectId>.GetInstance(out var projectsToRemove);
+                using var _4 = ArrayBuilder<ProjectId>.GetInstance(out var projectsToRemove);
 
                 // removed project
                 foreach (var (projectId, _) in oldProjectIdToStateChecksums)
@@ -549,7 +558,9 @@ await _assetProvider.GetAssetsAsync<DocumentStateChecksums, Dictionary<DocumentI
                 // 🔗 https://devdiv.visualstudio.com/DevDiv/_workitems/edit/1365014
                 if (newDocumentIdToStateChecksums.Count > 2)
                 {
-                    await _assetProvider.SynchronizeProjectAssetsAsync(projectChecksums, cancellationToken).ConfigureAwait(false);
+                    using var _ = ArrayBuilder<ProjectStateChecksums>.GetInstance(out var allProjectChecksums);
+                    allProjectChecksums.Add(projectChecksums);
+                    await _assetProvider.SynchronizeProjectAssetsAsync(allProjectChecksums, cancellationToken).ConfigureAwait(false);
                 }
 
                 return await UpdateDocumentsAsync(project, addDocuments, removeDocuments, oldDocumentIdToStateChecksums, newDocumentIdToStateChecksums, cancellationToken).ConfigureAwait(false);