[Dependency Scanner] Share ModuleDependenciesCache as a part of the CompilerInstance

This will allow individual module scans in batch-scanning mode to share the already-scanned Swift and Clang modules, and avoid instantiating a brand new Clang `DependencyScanningService` for each batch entry.

Performance-improvement anecdote:
A simple experiment which performs a batch scan of all modules involved in planning a build of SwiftPM (using SwiftPM in explicit module build mode), which includes 126 batch entries (scan entry-points), with 21 distinct Swift modules and 27 distinct Clang modules (many Clang modules are scanned multiple times, at different target versions), this modification reduces the total time taken from ~34 seconds to ~6.

Author: artemcm

include/swift/Frontend/Frontend.h

@@ -416,6 +416,10 @@ class CompilerInstance {
   std::unique_ptr<Lowering::TypeConverter> TheSILTypes;
   std::unique_ptr<DiagnosticVerifier> DiagVerifier;
 
+  /// A cache describing the set of inter-module dependencies that have been queried.
+  /// Null if not present.
+  std::unique_ptr<ModuleDependenciesCache> ModDepCache;
+
   /// Null if no tracker.
   std::unique_ptr<DependencyTracker> DepTracker;
   /// If there is no stats output directory by the time the
@@ -487,6 +491,8 @@ class CompilerInstance {
   DependencyTracker *getDependencyTracker() { return DepTracker.get(); }
   const DependencyTracker *getDependencyTracker() const { return DepTracker.get(); }
 
+  ModuleDependenciesCache *getModuleDependencyCache() { return ModDepCache.get(); }
+
   UnifiedStatsReporter *getStatsReporter() const { return Stats.get(); }
 
   /// Retrieve the main module containing the files being compiled.
@@ -552,6 +558,7 @@ class CompilerInstance {
   bool setUpVirtualFileSystemOverlays();
   void setUpLLVMArguments();
   void setUpDiagnosticOptions();
+  void setUpModuleDependencyCacheIfNeeded();
   bool setUpModuleLoaders();
   bool setUpInputs();
   bool setUpASTContextIfNeeded();

lib/Frontend/Frontend.cpp

@@ -321,11 +321,22 @@ void CompilerInstance::setupDependencyTrackerIfNeeded() {
   DepTracker = std::make_unique<DependencyTracker>(*collectionMode);
 }
 
+void CompilerInstance::setUpModuleDependencyCacheIfNeeded() {
+  const auto &Invocation = getInvocation();
+  const auto &opts = Invocation.getFrontendOptions();
+  if (opts.RequestedAction == FrontendOptions::ActionType::ScanDependencies ||
+      opts.RequestedAction == FrontendOptions::ActionType::ScanClangDependencies) {
+    ModDepCache = std::make_unique<ModuleDependenciesCache>();
+  }
+}
+
 bool CompilerInstance::setup(const CompilerInvocation &Invok) {
   Invocation = Invok;
 
   setupDependencyTrackerIfNeeded();
 
+  setUpModuleDependencyCacheIfNeeded();
+
   // If initializing the overlay file system fails there's no sense in
   // continuing because the compiler will read the wrong files.
   if (setUpVirtualFileSystemOverlays())

lib/FrontendTool/ScanDependencies.cpp

@@ -680,8 +680,10 @@ static bool scanModuleDependencies(CompilerInstance &instance,
 
   llvm::SetVector<ModuleDependencyID, std::vector<ModuleDependencyID>,
                   std::set<ModuleDependencyID>> allModules;
-  // Create the module dependency cache.
-  ModuleDependenciesCache cache;
+  // Retrieve the instance's module dependency cache.
+  ModuleDependenciesCache *cache = instance.getModuleDependencyCache();
+  assert(cache &&
+         "Dependency Scanner expected a ModuleDependenciesCache on a compiler instance.");
   InterfaceSubContextDelegateImpl ASTDelegate(ctx.SourceMgr, ctx.Diags,
                                               ctx.SearchPathOpts, ctx.LangOpts,
                                               ctx.ClangImporterOpts,
@@ -697,11 +699,11 @@ static bool scanModuleDependencies(CompilerInstance &instance,
   if (isClang) {
     // Loading the clang module using Clang importer.
     // This action will populate the cache with the main module's dependencies.
-    rootDeps = ctx.getModuleDependencies(moduleName, /*IsClang*/true, cache,
+    rootDeps = ctx.getModuleDependencies(moduleName, /*IsClang*/true, *cache,
                                          ASTDelegate);
   } else {
     // Loading the swift module's dependencies.
-    rootDeps = ctx.getSwiftModuleDependencies(moduleName, cache, ASTDelegate);
+    rootDeps = ctx.getSwiftModuleDependencies(moduleName, *cache, ASTDelegate);
   }
   if (!rootDeps.hasValue()) {
     // We cannot find the clang module, abort.
@@ -723,11 +725,11 @@ static bool scanModuleDependencies(CompilerInstance &instance,
        ++currentModuleIdx) {
     auto module = allModules[currentModuleIdx];
     auto discoveredModules =
-        resolveDirectDependencies(instance, module, cache, ASTDelegate);
+        resolveDirectDependencies(instance, module, *cache, ASTDelegate);
     allModules.insert(discoveredModules.begin(), discoveredModules.end());
   }
   // Write out the JSON description.
-  writeJSON(out, instance, cache, ASTDelegate, allModules.getArrayRef());
+  writeJSON(out, instance, *cache, ASTDelegate, allModules.getArrayRef());
   return false;
 }
 
@@ -889,9 +891,11 @@ bool swift::scanDependencies(CompilerInstance &instance) {
 
   allModules.insert({mainModuleName.str(), mainDependencies.getKind()});
 
-  // Create the module dependency cache.
-  ModuleDependenciesCache cache;
-  cache.recordDependencies(mainModuleName, std::move(mainDependencies));
+  // Retrieve the instance's module dependency cache.
+  ModuleDependenciesCache *cache = instance.getModuleDependencyCache();
+  assert(cache &&
+         "Dependency Scanner expected a ModuleDependenciesCache on a compiler instance.");
+  cache->recordDependencies(mainModuleName, std::move(mainDependencies));
 
   auto &ctx = instance.getASTContext();
   auto ModuleCachePath = getModuleCachePathFromClang(ctx
@@ -914,28 +918,28 @@ bool swift::scanDependencies(CompilerInstance &instance) {
        ++currentModuleIdx) {
     auto module = allModules[currentModuleIdx];
     auto discoveredModules =
-        resolveDirectDependencies(instance, module, cache, ASTDelegate);
+        resolveDirectDependencies(instance, module, *cache, ASTDelegate);
     allModules.insert(discoveredModules.begin(), discoveredModules.end());
   }
 
   // We have all explicit imports now, resolve cross import overlays.
   discoverCrosssImportOverlayDependencies(instance, mainModuleName,
-      /*All transitive dependencies*/allModules.getArrayRef().slice(1), cache,
+      /*All transitive dependencies*/allModules.getArrayRef().slice(1), *cache,
       ASTDelegate, [&](ModuleDependencyID id) {
     allModules.insert(id);
   });
 
   // Dignose cycle in dependency graph.
-  if (diagnoseCycle(instance, cache, /*MainModule*/allModules.front(), ASTDelegate))
+  if (diagnoseCycle(instance, *cache, /*MainModule*/allModules.front(), ASTDelegate))
     return true;
 
   // Write out the JSON description.
-  writeJSON(out, instance, cache, ASTDelegate, allModules.getArrayRef());
+  writeJSON(out, instance, *cache, ASTDelegate, allModules.getArrayRef());
 
   // Update the dependency tracker.
   if (auto depTracker = instance.getDependencyTracker()) {
     for (auto module : allModules) {
-      auto deps = cache.findDependencies(module.first, module.second);
+      auto deps = cache->findDependencies(module.first, module.second);
       if (!deps)
         continue;