changes

MSBuild.CompilerCache/CacheCombiner.cs

@@ -1,3 +1,5 @@
+using IRefCache = MSBuild.CompilerCache.ICacheBase<MSBuild.CompilerCache.CacheKey, MSBuild.CompilerCache.RefDataWithOriginalExtract>;
+
 namespace MSBuild.CompilerCache;
 
 public class CacheCombiner<TKey, TValue> : ICacheBase<TKey, TValue> where TValue : class
@@ -47,4 +49,12 @@ public bool Set(TKey key, TValue value)
             return false;
         }
     }
-}
+}
+
+public static class CacheCombiner
+{
+    public static ICacheBase<TKey,TValue> Combine<TKey,TValue>(ICacheBase<TKey, TValue> first, ICacheBase<TKey,TValue> second) where TValue : class
+    {
+        return new CacheCombiner<TKey, TValue>(first, second);
+    }    
+}

MSBuild.CompilerCache/CompilationResultsCache.cs

@@ -124,22 +124,15 @@ public interface ICompilationResultsCache
     /// </summary>
     /// <param name="key"></param>
     /// <param name="fullExtract"></param>
-    /// <param name="resultZip"></param>
-    void Set(CacheKey key, FullExtract fullExtract, FileInfo resultZip);
+    /// <param name="resultZipToBeMoved"></param>
+    void Set(CacheKey key, FullExtract fullExtract, FileInfo resultZipToBeMoved);
     string? Get(CacheKey key);
 }
 
 public class CompilationResultsCache : ICompilationResultsCache
 {
     private readonly string _baseCacheDir;
 
-    private static readonly JsonSerializerOptions JsonOptions = new JsonSerializerOptions()
-    {
-        WriteIndented = true
-    };
-
-    private JsonWriterOptions _writeOptions;
-
     public CompilationResultsCache(string baseCacheDir)
     {
         _baseCacheDir = baseCacheDir;
@@ -162,11 +155,18 @@ public bool Exists(CacheKey key)
     /// <param name="destination"></param>
     /// <param name="throwIfDestinationExists">If true, will throw if the destination already exists</param>
     /// <returns></returns>
-    public static bool AtomicCopy(string source, string destination, bool throwIfDestinationExists = true)
+    public static bool AtomicCopy(string source, string destination, bool throwIfDestinationExists = true, bool moveInsteadOfCopy = false)
     {
         var dir = Path.GetDirectoryName(destination)!;
         var tmpDestination = Path.Combine(dir, $".__tmp_{Guid.NewGuid()}");
-        File.Copy(source, tmpDestination);
+        if (moveInsteadOfCopy)
+        {
+            File.Move(source, tmpDestination);
+        }
+        else
+        {
+            File.Copy(source, tmpDestination);
+        }
         try
         {
             File.Move(tmpDestination, destination, overwrite: false);
@@ -201,27 +201,27 @@ public CacheKey[] GetAllExistingKeys()
             .ToArray();
     }
 
-    public void Set(CacheKey key, FullExtract fullExtract, FileInfo resultZip)
+    public void Set(CacheKey key, FullExtract fullExtract, FileInfo resultZipToBeMoved)
     {
         var dir = new DirectoryInfo(CacheDir(key));
         dir.Create();
-
         var extractPath = ExtractPath(key);
-
-        var outputPath = Path.Combine(dir.FullName, resultZip.Name);
+        var outputPath = Path.Combine(dir.FullName, resultZipToBeMoved.Name);
+
         if (!File.Exists(outputPath))
         {
-            AtomicCopy(resultZip.FullName, outputPath, throwIfDestinationExists: false);
+            AtomicCopy(resultZipToBeMoved.FullName, outputPath, throwIfDestinationExists: false, moveInsteadOfCopy: true);
         }
 
         if (!File.Exists(extractPath))
         {
+            // TODO Serialise directly to the cache dir (as a tmp file)
             using var tmpFile = new TempFile();
             {
                 using var fs = tmpFile.File.OpenWrite();
                 JsonSerializer.Serialize(fs, fullExtract, FullExtractJsonContext.Default.FullExtract);
             }
-            AtomicCopy(tmpFile.FullName, extractPath, throwIfDestinationExists: false);
+            AtomicCopy(tmpFile.FullName, extractPath, throwIfDestinationExists: false, moveInsteadOfCopy: true);
         }
     }
 

MSBuild.CompilerCache/LocatorAndPopulator.cs

@@ -1,3 +1,5 @@
+using System.Text.Json.Serialization;
+
 namespace MSBuild.CompilerCache;
 
 using System.Collections.Immutable;
@@ -59,15 +61,16 @@ public class LocatorAndPopulator
         Action<string>? logTime = null)
     {
         using var fs = File.OpenRead(configPath);
-        var config = JsonSerializer.Deserialize<Config>(fs);
+        var config = JsonSerializer.Deserialize<Config>(fs)!;
         //logTime?.Invoke("Config deserialized");
         var cache = new CompilationResultsCache(config.CacheDir);
         var refCache = new RefCache(config.InferRefCacheDir());
+        var combinedRefCache = CacheCombiner.Combine(_inMemoryRefCache, refCache);
         //logTime?.Invoke("Finish");
         var fileBasedFileHashCache = new FileHashCache(config.InferFileHashCacheDir());
         var fileHashCache = new CacheCombiner<FileHashCacheKey, string>(_inMemoryFileHashCache, fileBasedFileHashCache);
         var hasher = HasherFactory.CreateHash(config.Hasher);
-        return (config, cache, refCache, fileHashCache, hasher);
+        return (config, cache, combinedRefCache, fileHashCache, hasher);
     }
 
     // These fields are populated in the 'Locate' call and used in a subsequent 'Populate' call
@@ -373,8 +376,9 @@ bool FileChanged(FileHashCacheKey file)
 
         var changedFile =
             _localInputs.Files
-                .Chunk(Math.Max(1, _localInputs.Files.Length / 4))
+                .Chunk(Math.Max(1, _localInputs.Files.Length / 8))
                 .AsParallel()
+                .WithExecutionMode(ParallelExecutionMode.ForceParallelism)
                 .Select(files =>
                 {
                     foreach (var file in files)
@@ -393,9 +397,18 @@ bool FileChanged(FileHashCacheKey file)
 
     public UseOrPopulateResult PopulateCache(TaskLoggingHelper log, Action<string>? logTime = null)
     {
+        // Trigger RefTrimmer for newly-built dlls/exe files - should speed up builds of dependent projects,
+        // and avoid any duplicate calculations due to multiple dependants redoing the same thing if timings are bad.
+
         //logTime?.Invoke("Start");
         var postCompilationTimeUtc = DateTime.UtcNow;
 
+        var meta = GetCompilationMetadata(postCompilationTimeUtc);
+        var stuff = new AllCompilationMetadata(Metadata: meta, LocalInputs: _localInputs);
+        //logTime?.Invoke("Got stuff");
+        using var tmpDir = new DisposableDir();
+        var outputZip = BuildOutputsZip(tmpDir, _decomposed.OutputsToCache, stuff, Utils.DefaultHasher, log);
+
         var changedFile = CheckInputsConsistency(log);
 
         //logTime?.Invoke("Calculated local inputs with hash");
@@ -408,11 +421,6 @@ public UseOrPopulateResult PopulateCache(TaskLoggingHelper log, Action<string>?
             //logTime?.Invoke("Hashes match");
             log.LogMessage(MessageImportance.Normal,
                 $"CompilationCache - copying {_decomposed.OutputsToCache.Length} files from output to cache");
-            var meta = GetCompilationMetadata(postCompilationTimeUtc);
-            var stuff = new AllCompilationMetadata(Metadata: meta, LocalInputs: _localInputs);
-            //logTime?.Invoke("Got stuff");
-            using var tmpDir = new DisposableDir();
-            var outputZip = BuildOutputsZip(tmpDir, _decomposed.OutputsToCache, stuff, Utils.DefaultHasher, log);
             //logTime?.Invoke("Outputs zip created");
             _cache.Set(_locateResult.CacheKey!.Value, _extract, outputZip);
             //logTime?.Invoke("cache entry set");
@@ -432,6 +440,15 @@ public static FileInfo BuildOutputsZip(DirectoryInfo baseTmpDir, OutputItem[] it
     {
         var outputsDir = baseTmpDir.CreateSubdirectory("outputs_zip_building");
 
+        System.Threading.Tasks.Task.Run(() =>
+        {
+            var metaPath = outputsDir.CombineAsFile("__inputs.json").FullName;
+            {
+                using var fs = File.OpenWrite(metaPath);
+                JsonSerializer.Serialize(fs, metadata, AllCompilationMetadataJsonContext.Default.AllCompilationMetadata);
+            }
+        });
+
         var outputExtracts =
             items.Select(item =>
             {
@@ -444,26 +461,27 @@ public static FileInfo BuildOutputsZip(DirectoryInfo baseTmpDir, OutputItem[] it
 
         var hashForFileName = Utils.ObjectToHash(outputExtracts, hasher);
 
-        var jsonOptions = new JsonSerializerOptions()
-        {
-            WriteIndented = true
-        };
-
         var outputsExtractJsonPath = outputsDir.CombineAsFile("__outputs.json").FullName;
         {
             using var fs = File.OpenWrite(outputsExtractJsonPath);
-            JsonSerializer.Serialize(fs, outputExtracts, jsonOptions);
-        }
-
-        var metaPath = outputsDir.CombineAsFile("__inputs.json").FullName;
-        {
-            using var fs = File.OpenWrite(metaPath);
-            JsonSerializer.Serialize(fs, metadata, jsonOptions);
+            JsonSerializer.Serialize(fs, outputExtracts, OutputExtractsJsonContext.Default.FileExtractArray);
         }
 
         var tempZipPath = baseTmpDir.CombineAsFile($"{hashForFileName}.zip");
         ZipFile.CreateFromDirectory(outputsDir.FullName, tempZipPath.FullName,
             CompressionLevel.NoCompression, includeBaseDirectory: false);
         return tempZipPath;
     }
-}
+}
+
+
+[JsonSerializable(typeof(FileExtract[]))]
+[JsonSourceGenerationOptions(WriteIndented = true)]
+public partial class OutputExtractsJsonContext : JsonSerializerContext
+{ }
+
+
+[JsonSerializable(typeof(AllCompilationMetadata))]
+[JsonSourceGenerationOptions(WriteIndented = true)]
+public partial class AllCompilationMetadataJsonContext : JsonSerializerContext
+{ }

MSBuild.CompilerCache/RefCache.cs

@@ -3,10 +3,12 @@
 namespace MSBuild.CompilerCache;
 
 using System.Collections.Concurrent;
-using System.Collections.Immutable;
-using System.Text;
 using IRefCache = ICacheBase<CacheKey, RefDataWithOriginalExtract>;
 
+/// <summary>
+/// File-based implementation of <see cref="IRefCache"/> for storing information about trimmed dlls and their hashes.
+/// Stores all entries in a single directory, one .json file per entry.
+/// </summary>
 public class RefCache : IRefCache
 {
     private readonly string _cacheDir;
@@ -88,18 +90,5 @@ public bool Set(CacheKey key, RefDataWithOriginalExtract data)
     }
 }
 
-public class InMemoryRefCache : IRefCache
-{
-    private readonly ConcurrentDictionary<CacheKey, RefDataWithOriginalExtract> _cache =
-        new ConcurrentDictionary<CacheKey, RefDataWithOriginalExtract>();
-
-    public bool Exists(CacheKey key) => _cache.ContainsKey(key);
-
-    public RefDataWithOriginalExtract? Get(CacheKey key)
-    {
-        _cache.TryGetValue(key, out var value);
-        return value;
-    }
-
-    public bool Set(CacheKey key, RefDataWithOriginalExtract data) => _cache.TryAdd(key, data);
-}
+public class InMemoryRefCache : DictionaryBasedCache<CacheKey, RefDataWithOriginalExtract>
+{ }