Merge pull request #356 from artemcm/DependencyOracle

[Explicit Module Builds] Introduce an InterModuleDependencyOracle abstraction for tracking and sharing dependency scanning results

Author: artemcm

Sources/SwiftDriver/CMakeLists.txt

@@ -9,12 +9,13 @@
 add_library(SwiftDriver
   "Explicit Module Builds/ClangModuleBuildJobCache.swift"
   "Explicit Module Builds/ClangVersionedDependencyResolution.swift"
-  "Explicit Module Builds/CommonDependencyGraphOperations.swift"
   "Explicit Module Builds/ExplicitDependencyBuildPlanner.swift"
-  "Explicit Module Builds/InterModuleDependencyGraph.swift"
   "Explicit Module Builds/ModuleDependencyScanning.swift"
   "Explicit Module Builds/PlaceholderDependencyResolution.swift"
   "Explicit Module Builds/SerializableModuleArtifacts.swift"
+  "Explicit Module Builds/Inter Module Dependencies/CommonDependencyOperations.swift"
+  "Explicit Module Builds/Inter Module Dependencies/InterModuleDependencyGraph.swift"
+  "Explicit Module Builds/Inter Module Dependencies/InterModuleDependencyOracle.swift"
 
   Driver/CompilerMode.swift
   Driver/DebugInfo.swift

Sources/SwiftDriver/Driver/Driver.swift

@@ -256,11 +256,21 @@ public struct Driver {
   /// dependencies in parallel builds.
   var forceEmitModuleBeforeCompile: Bool = false
 
+  // FIXME: We should soon be able to remove this from being in the Driver's state.
+  // Its only remaining use outside of actual dependency build planning is in
+  // command-line input option generation for the explicit main module compile job.
   /// Planner for constructing module build jobs using Explicit Module Builds.
-  /// Constructed during the planning phase only when all modules will be prebuilt and treated
-  /// as explicit by the various compilation jobs.
+  /// Constructed during the planning phase only when all module dependencies will be prebuilt and treated
+  /// as explicit inputs by the various compilation jobs.
   @_spi(Testing) public var explicitDependencyBuildPlanner: ExplicitDependencyBuildPlanner? = nil
 
+  /// An oracle for querying inter-module dependencies
+  /// Can either be an argument to the driver in many-module contexts where dependency information
+  /// is shared across many targets; otherwise, a new instance is created by the driver itself.
+  @_spi(Testing) public let interModuleDependencyOracle: InterModuleDependencyOracle
+
+  // TODO: Once the clients have transitioned to using the InterModuleDependencyOracle API,
+  // this must convey information about the externally-prebuilt targets only
   /// All external artifacts a build system (e.g. SwiftPM) may pass in as input to the explicit
   /// build of the current module. Consists of a map of externally-built targets, and a map of all previously
   /// discovered/scanned modules and their infos.
@@ -311,15 +321,23 @@ public struct Driver {
     diagnosticsEngine: DiagnosticsEngine = DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler]),
     fileSystem: FileSystem = localFileSystem,
     executor: DriverExecutor,
-    externalBuildArtifacts: ExternalBuildArtifacts? = nil
+    // FIXME: Duplication with externalBuildArtifacts and externalTargetModulePathMap
+    // is a temporary backwards-compatibility shim to help transition SwiftPM to the new API
+    externalBuildArtifacts: ExternalBuildArtifacts? = nil,
+    externalTargetModulePathMap: ExternalTargetModulePathMap? = nil,
+    interModuleDependencyOracle: InterModuleDependencyOracle? = nil
   ) throws {
     self.env = env
     self.fileSystem = fileSystem
 
     self.diagnosticEngine = diagnosticsEngine
     self.executor = executor
 
-    self.externalBuildArtifacts = externalBuildArtifacts
+    if let externalArtifacts = externalBuildArtifacts {
+      self.externalBuildArtifacts = externalArtifacts
+    } else if let externalTargetPaths = externalTargetModulePathMap {
+      self.externalBuildArtifacts = (externalTargetPaths, [:])
+    }
 
     if case .subcommand = try Self.invocationRunMode(forArgs: args).mode {
       throw Error.subcommandPassedToDriver
@@ -370,7 +388,24 @@ public struct Driver {
         guard let modTime = try? fileSystem
           .getFileInfo($0.file).modTime else { return nil }
         return ($0, modTime)
-      })
+    })
+
+    // Create an instance of an inter-module dependency oracle, if the driver's
+    // client did not provide one. The clients are expected to provide an oracle
+    // when they wish to share module dependency information across targets.
+    if let dependencyOracle = interModuleDependencyOracle {
+      self.interModuleDependencyOracle = dependencyOracle
+    } else {
+      self.interModuleDependencyOracle = InterModuleDependencyOracle()
+
+      // This is a shim for backwards-compatibility with ModuleInfoMap-based API
+      // used by SwiftPM
+      if let externalArtifacts = externalBuildArtifacts {
+        if !externalArtifacts.1.isEmpty {
+          try self.interModuleDependencyOracle.mergeModules(from: externalArtifacts.1)
+        }
+      }
+    }
 
     do {
       let outputFileMap: OutputFileMap?

Sources/SwiftDriver/Explicit Module Builds/ClangVersionedDependencyResolution.swift

@@ -108,24 +108,35 @@ private extension InterModuleDependencyGraph {
                pathPCMArtSet: Set<[String]>,
                pcmArgSetMap: inout [ModuleDependencyId : Set<[String]>])
     throws {
+      guard let moduleInfo = modules[moduleId] else {
+        throw Driver.Error.missingModuleDependency(moduleId.moduleName)
+      }
       switch moduleId {
         case .swift:
+          guard case .swift(let swiftModuleDetails) = moduleInfo.details else {
+            throw Driver.Error.malformedModuleDependency(moduleId.moduleName,
+                                                         "no Swift `details` object")
+          }
           // Add extraPCMArgs of the visited node to the current path set
           // and proceed to visit all direct dependencies
-          let modulePCMArgs = try swiftModulePCMArgs(of: moduleId)
+          let modulePCMArgs = swiftModuleDetails.extraPcmArgs
           var newPathPCMArgSet = pathPCMArtSet
           newPathPCMArgSet.insert(modulePCMArgs)
-          for dependencyId in try moduleInfo(of: moduleId).directDependencies! {
+          for dependencyId in moduleInfo.directDependencies! {
             try visit(dependencyId,
                       pathPCMArtSet: newPathPCMArgSet,
                       pcmArgSetMap: &pcmArgSetMap)
           }
         case .clang:
+          guard case .clang(let clangModuleDetails) = moduleInfo.details else {
+            throw Driver.Error.malformedModuleDependency(moduleId.moduleName,
+                                                         "no Clang `details` object")
+          }
           // The details of this module contain information on which sets of PCMArgs are already
           // captured in the described dependencies of this module. Only re-scan at PCMArgs not
           // already captured.
-          let moduleDetails = try clangModuleDetails(of: moduleId)
-          let alreadyCapturedPCMArgs = moduleDetails.dependenciesCapturedPCMArgs ?? Set<[String]>()
+          let alreadyCapturedPCMArgs =
+            clangModuleDetails.dependenciesCapturedPCMArgs ?? Set<[String]>()
           let newPCMArgSet = pathPCMArtSet.filter { !alreadyCapturedPCMArgs.contains($0) }
           // Add current path's PCMArgs to the SetMap and stop traversal
           if pcmArgSetMap[moduleId] != nil {
@@ -138,7 +149,7 @@ private extension InterModuleDependencyGraph {
           // We can rely on the fact that this pre-built module already has its
           // versioned-PCM dependencies satisfied, so we do not need to add additional
           // arguments. Proceed traversal to its dependencies.
-          for dependencyId in try moduleInfo(of: moduleId).directDependencies! {
+          for dependencyId in moduleInfo.directDependencies! {
             try visit(dependencyId,
                       pathPCMArtSet: pathPCMArtSet,
                       pcmArgSetMap: &pcmArgSetMap)
@@ -159,57 +170,19 @@ private extension InterModuleDependencyGraph {
                                                         [ModuleDependencyId : Set<[String]>]
   ) throws {
     for (moduleId, newPCMArgs) in pcmArgSetMap {
-      var moduleDetails = try clangModuleDetails(of: moduleId)
-      if moduleDetails.dependenciesCapturedPCMArgs == nil {
-        moduleDetails.dependenciesCapturedPCMArgs = Set<[String]>()
+      guard let moduleInfo = modules[moduleId] else {
+        throw Driver.Error.missingModuleDependency(moduleId.moduleName)
+      }
+      guard case .clang(var clangModuleDetails) = moduleInfo.details else {
+        throw Driver.Error.malformedModuleDependency(moduleId.moduleName,
+                                                     "no Clang `details` object")
       }
-      newPCMArgs.forEach { moduleDetails.dependenciesCapturedPCMArgs!.insert($0) }
-      modules[moduleId]!.details = .clang(moduleDetails)
+      if clangModuleDetails.dependenciesCapturedPCMArgs == nil {
+        clangModuleDetails.dependenciesCapturedPCMArgs = Set<[String]>()
+      }
+      newPCMArgs.forEach { clangModuleDetails.dependenciesCapturedPCMArgs!.insert($0) }
+      modules[moduleId]!.details = .clang(clangModuleDetails)
     }
   }
 }
 
-public extension InterModuleDependencyGraph {
-  /// Given two moduleInfos of clang modules, merge them by combining their directDependencies and
-  /// dependenciesCapturedPCMArgs and sourceFiles fields. These fields may differ across the same module
-  /// scanned at different PCMArgs (e.g. -target option).
-  static func mergeClangModuleInfoDependencies(_ firstInfo: ModuleInfo, _ secondInfo:ModuleInfo
-  ) -> ModuleInfo {
-    guard case .clang(let firstDetails) = firstInfo.details,
-          case .clang(let secondDetails) = secondInfo.details
-    else {
-      fatalError("mergeClangModules expected two valid ClangModuleDetails objects.")
-    }
-
-    // As far as their dependencies go, these module infos are identical
-    if firstInfo.directDependencies == secondInfo.directDependencies,
-       firstDetails.dependenciesCapturedPCMArgs == secondDetails.dependenciesCapturedPCMArgs,
-       firstInfo.sourceFiles == secondInfo.sourceFiles {
-      return firstInfo
-    }
-
-    // Create a new moduleInfo that represents this module with combined dependency information
-    let firstModuleSources = firstInfo.sourceFiles ?? []
-    let secondModuleSources = secondInfo.sourceFiles ?? []
-    let combinedSourceFiles = Array(Set(firstModuleSources + secondModuleSources))
-
-    let firstModuleDependencies = firstInfo.directDependencies ?? []
-    let secondModuleDependencies = secondInfo.directDependencies ?? []
-    let combinedDependencies = Array(Set(firstModuleDependencies + secondModuleDependencies))
-
-    let firstModuleCapturedPCMArgs = firstDetails.dependenciesCapturedPCMArgs ?? Set<[String]>()
-    let secondModuleCapturedPCMArgs = secondDetails.dependenciesCapturedPCMArgs ?? Set<[String]>()
-    let combinedCapturedPCMArgs = firstModuleCapturedPCMArgs.union(secondModuleCapturedPCMArgs)
-
-    let combinedModuleDetails =
-      ClangModuleDetails(moduleMapPath: firstDetails.moduleMapPath,
-                         dependenciesCapturedPCMArgs: combinedCapturedPCMArgs,
-                         contextHash: firstDetails.contextHash,
-                         commandLine: firstDetails.commandLine)
-
-    return ModuleInfo(modulePath: firstInfo.modulePath,
-                      sourceFiles: combinedSourceFiles,
-                      directDependencies: combinedDependencies,
-                      details: .clang(combinedModuleDetails))
-  }
-}

Sources/SwiftDriver/Explicit Module Builds/ExplicitDependencyBuildPlanner.swift

@@ -15,6 +15,9 @@ import Foundation
 
 /// A map from a module identifier to a path to its .swiftmodule file.
 public typealias ExternalTargetModulePathMap = [ModuleDependencyId: AbsolutePath]
+
+// FIXME: ExternalBuildArtifacts is a temporary backwards-compatibility shim
+// to help transition SwiftPM to the new API.
 /// A tuple all external artifacts a build system may pass in as input to the explicit build of the current module
 /// Consists of a map of externally-built targets, and a map of all previously discovered/scanned modules.
 public typealias ExternalBuildArtifacts = (ExternalTargetModulePathMap, ModuleInfoMap)
@@ -513,11 +516,3 @@ internal extension InterModuleDependencyGraph {
     return String(data: contents, encoding: .utf8)!
   }
 }
-
-// To keep the ExplicitDependencyBuildPlanner an implementation detail, provide an API
-// to access the dependency graph
-extension Driver {
-  public var interModuleDependencyGraph: InterModuleDependencyGraph? {
-    return explicitDependencyBuildPlanner?.dependencyGraph
-  }
-}

Sources/SwiftDriver/Explicit Module Builds/CommonDependencyGraphOperations.swift renamed to Sources/SwiftDriver/Explicit Module Builds/Inter Module Dependencies/CommonDependencyOperations.swift

@@ -1,4 +1,4 @@
-//===-------------- CommonDependencyGraphOperations.swift -----------------===//
+//===----------------- CommonDependencyOperations.swift -------------------===//
 //
 // This source file is part of the Swift.org open source project
 //
@@ -10,9 +10,31 @@
 //
 //===----------------------------------------------------------------------===//
 
-public extension InterModuleDependencyGraph {
+@_spi(Testing) public extension InterModuleDependencyOracle {
   /// An API to allow clients to accumulate InterModuleDependencyGraphs across mutiple main modules/targets
   /// into a single collection of discovered modules.
+  func mergeModules(from dependencyGraph: InterModuleDependencyGraph) throws {
+    try self.lock.withLock {
+      for (moduleId, moduleInfo) in dependencyGraph.modules {
+        try InterModuleDependencyGraph.mergeModule(moduleId, moduleInfo, into: &modules)
+      }
+    }
+  }
+
+  // This is a backwards-compatibility shim to handle existing ModuleInfoMap-based API
+  // used by SwiftPM
+  func mergeModules(from moduleInfoMap: ModuleInfoMap) throws {
+    try self.lock.withLock {
+      for (moduleId, moduleInfo) in moduleInfoMap {
+        try InterModuleDependencyGraph.mergeModule(moduleId, moduleInfo, into: &modules)
+      }
+    }
+  }
+}
+
+public extension InterModuleDependencyGraph {
+  // This is a shim for backwards-compatibility with existing API used by SwiftPM.
+  // TODO: After SwiftPM switches to using the oracle, this should be deleted.
   static func mergeModules(
     from dependencyGraph: InterModuleDependencyGraph,
     into discoveredModules: inout ModuleInfoMap
@@ -23,7 +45,8 @@ public extension InterModuleDependencyGraph {
   }
 }
 
-internal extension InterModuleDependencyGraph {
+
+@_spi(Testing) public extension InterModuleDependencyGraph {
   /// Merge a module with a given ID and Info into a ModuleInfoMap
   static func mergeModule(_ moduleId: ModuleDependencyId,
                           _ moduleInfo: ModuleInfo,
@@ -114,4 +137,47 @@ internal extension InterModuleDependencyGraph {
       }
     }
   }
+
+  /// Given two moduleInfos of clang modules, merge them by combining their directDependencies and
+  /// dependenciesCapturedPCMArgs and sourceFiles fields. These fields may differ across the same module
+  /// scanned at different PCMArgs (e.g. -target option).
+  static func mergeClangModuleInfoDependencies(_ firstInfo: ModuleInfo, _ secondInfo:ModuleInfo
+  ) -> ModuleInfo {
+    guard case .clang(let firstDetails) = firstInfo.details,
+          case .clang(let secondDetails) = secondInfo.details
+    else {
+      fatalError("mergeClangModules expected two valid ClangModuleDetails objects.")
+    }
+
+    // As far as their dependencies go, these module infos are identical
+    if firstInfo.directDependencies == secondInfo.directDependencies,
+       firstDetails.dependenciesCapturedPCMArgs == secondDetails.dependenciesCapturedPCMArgs,
+       firstInfo.sourceFiles == secondInfo.sourceFiles {
+      return firstInfo
+    }
+
+    // Create a new moduleInfo that represents this module with combined dependency information
+    let firstModuleSources = firstInfo.sourceFiles ?? []
+    let secondModuleSources = secondInfo.sourceFiles ?? []
+    let combinedSourceFiles = Array(Set(firstModuleSources + secondModuleSources))
+
+    let firstModuleDependencies = firstInfo.directDependencies ?? []
+    let secondModuleDependencies = secondInfo.directDependencies ?? []
+    let combinedDependencies = Array(Set(firstModuleDependencies + secondModuleDependencies))
+
+    let firstModuleCapturedPCMArgs = firstDetails.dependenciesCapturedPCMArgs ?? Set<[String]>()
+    let secondModuleCapturedPCMArgs = secondDetails.dependenciesCapturedPCMArgs ?? Set<[String]>()
+    let combinedCapturedPCMArgs = firstModuleCapturedPCMArgs.union(secondModuleCapturedPCMArgs)
+
+    let combinedModuleDetails =
+      ClangModuleDetails(moduleMapPath: firstDetails.moduleMapPath,
+                         dependenciesCapturedPCMArgs: combinedCapturedPCMArgs,
+                         contextHash: firstDetails.contextHash,
+                         commandLine: firstDetails.commandLine)
+
+    return ModuleInfo(modulePath: firstInfo.modulePath,
+                      sourceFiles: combinedSourceFiles,
+                      directDependencies: combinedDependencies,
+                      details: .clang(combinedModuleDetails))
+  }
 }