[Explicit Module Builds] Emit a JSON file that specifies details of explicit Swift module dependencies

artemcm · artemcm · commit c4b99b8416ce · 2020-06-23T16:18:13.000-07:00
This PR implements the interface expected by the frontend, as built in: swiftlang/swift#32355 swiftlang/swift#32450 Swift module's explicit Swift module dependencies are now encoded in a JSON file, passed with `-explicit-swift-module-map-file` Resolves rdar://problem/64533451
diff --git a/Sources/SwiftDriver/CMakeLists.txt b/Sources/SwiftDriver/CMakeLists.txt
@@ -11,6 +11,7 @@ add_library(SwiftDriver
   "Explicit Module Builds/ExplicitModuleBuildHandler.swift"   
   "Explicit Module Builds/InterModuleDependencyGraph.swift"
   "Explicit Module Builds/ModuleDependencyScanning.swift"
+  "Explicit Module Builds/SwiftModuleArtifacts.swift"
 
   Driver/CompilerMode.swift
   Driver/DebugInfo.swift
diff --git a/Sources/SwiftDriver/Explicit Module Builds/ExplicitModuleBuildHandler.swift b/Sources/SwiftDriver/Explicit Module Builds/ExplicitModuleBuildHandler.swift
@@ -31,9 +31,19 @@ public struct ExplicitModuleBuildHandler {
   /// The toolchain to be used for frontend job generation.
   private let toolchain: Toolchain
 
-  public init(dependencyGraph: InterModuleDependencyGraph, toolchain: Toolchain) throws {
+  /// The file system which we should interact with.
+  private let fileSystem: FileSystem
+
+  /// Path to the directory that will contain the temporary files.
+  /// e.g. Explicit Swift module artifact files
+  private let temporaryDirectory: AbsolutePath
+
+  public init(dependencyGraph: InterModuleDependencyGraph, toolchain: Toolchain,
+              fileSystem: FileSystem) throws {
     self.dependencyGraph = dependencyGraph
     self.toolchain = toolchain
+    self.fileSystem = fileSystem
+    self.temporaryDirectory = try determineTempDirectory()
   }
 
   /// Generate build jobs for all dependencies of the main module.
@@ -97,17 +107,10 @@ public struct ExplicitModuleBuildHandler {
   /// - Generate Job: S1
   ///
   mutating public func generateExplicitModuleDependenciesBuildJobs() throws -> [Job] {
-    for moduleId in dependencyGraph.mainModule.directDependencies {
-      switch moduleId {
-        case .swift:
-          try genSwiftModuleBuildJob(moduleId: moduleId)
-        case .clang:
-          try genClangModuleBuildJob(moduleId: moduleId,
-                                     pcmArgs: try dependencyGraph.swiftModulePCMArgs(
-                                      of: .swift(dependencyGraph.mainModuleName)))
-      }
-    }
-
+    var mainModuleInputs: [TypedVirtualPath] = []
+    var mainModuleCommandLine: [Job.ArgTemplate] = []
+    try resolveMainModuleDependencies(inputs: &mainModuleInputs,
+                                      commandLine: &mainModuleCommandLine)
     return Array(swiftModuleBuildCache.values) + clangTargetModuleBuildCache.allJobs
   }
 
@@ -167,7 +170,6 @@ public struct ExplicitModuleBuildHandler {
   mutating private func genClangModuleBuildJob(moduleId: ModuleDependencyId,
                                                pcmArgs: [String]) throws {
     let moduleInfo = try dependencyGraph.moduleInfo(of: moduleId)
-
     var inputs: [TypedVirtualPath] = []
     var outputs: [TypedVirtualPath] = []
     var commandLine: [Job.ArgTemplate] = []
@@ -207,6 +209,19 @@ public struct ExplicitModuleBuildHandler {
     )
   }
 
+  /// Store the output file artifacts for a given module in a JSON file, return the file's path.
+  private func serializeModuleDependencies(for moduleId: ModuleDependencyId,
+                                           dependencyArtifacts: [SwiftModuleArtifactInfo]
+  ) throws -> AbsolutePath {
+    let dependencyFilePath =
+      temporaryDirectory.appending(component: "\(moduleId.moduleName)-dependencies.json")
+    let encoder = JSONEncoder()
+    encoder.outputFormatting = [.prettyPrinted]
+    let contents = try encoder.encode(dependencyArtifacts)
+    try fileSystem.writeFileContents(dependencyFilePath, bytes: ByteString(contents))
+    return dependencyFilePath
+  }
+
   /// For the specified module, update the given command line flags and inputs
   /// to use explicitly-built module dependencies.
   /// 
@@ -220,24 +235,45 @@ public struct ExplicitModuleBuildHandler {
     // Prohibit the frontend from implicitly building textual modules into binary modules.
     commandLine.appendFlags("-disable-implicit-swift-modules", "-Xcc", "-Xclang", "-Xcc",
                             "-fno-implicit-modules")
+    var swiftDependencyArtifacts: [SwiftModuleArtifactInfo] = []
     try addModuleDependencies(moduleId: moduleId, pcmArgs: pcmArgs, inputs: &inputs,
-                              commandLine: &commandLine)
+                              commandLine: &commandLine,
+                              swiftDependencyArtifacts: &swiftDependencyArtifacts)
+
+    if !swiftDependencyArtifacts.isEmpty {
+      let dependencyFile = try serializeModuleDependencies(for: moduleId,
+                                                           dependencyArtifacts: swiftDependencyArtifacts)
+      commandLine.appendFlag("-explicit-swift-module-map-file")
+      commandLine.appendPath(dependencyFile)
+      inputs.append(TypedVirtualPath(file: try VirtualPath(path: dependencyFile.pathString),
+                                     type: .jsonSwiftArtifacts))
+      // Each individual moduel binary is still an "input" to ensure the build system gets the
+      // order correctly.
+      for dependencyModule in swiftDependencyArtifacts {
+        inputs.append(TypedVirtualPath(file: try VirtualPath(path: dependencyModule.modulePath),
+                                       type: .swiftModule))
+      }
+    }
   }
 
   /// Add a specific module dependency as an input and a corresponding command
   /// line flag. Dispatches to clang and swift-specific variants.
   mutating private func addModuleDependencies(moduleId: ModuleDependencyId,
                                               pcmArgs: [String],
                                               inputs: inout [TypedVirtualPath],
-                                              commandLine: inout [Job.ArgTemplate]) throws {
+                                              commandLine: inout [Job.ArgTemplate],
+                                              swiftDependencyArtifacts: inout [SwiftModuleArtifactInfo]
+  ) throws {
     for dependencyId in try dependencyGraph.moduleInfo(of: moduleId).directDependencies {
       switch dependencyId {
         case .swift:
           try addSwiftModuleDependency(moduleId: moduleId, dependencyId: dependencyId,
-                                       pcmArgs: pcmArgs, inputs: &inputs, commandLine: &commandLine)
+                                       pcmArgs: pcmArgs, inputs: &inputs, commandLine: &commandLine,
+                                       swiftDependencyArtifacts: &swiftDependencyArtifacts)
         case .clang:
           try addClangModuleDependency(moduleId: moduleId, dependencyId: dependencyId,
-                                       pcmArgs: pcmArgs, inputs: &inputs, commandLine: &commandLine)
+                                       pcmArgs: pcmArgs, inputs: &inputs, commandLine: &commandLine,
+                                       swiftDependencyArtifacts: &swiftDependencyArtifacts)
       }
     }
   }
@@ -250,7 +286,9 @@ public struct ExplicitModuleBuildHandler {
                                                  dependencyId: ModuleDependencyId,
                                                  pcmArgs: [String],
                                                  inputs: inout [TypedVirtualPath],
-                                                 commandLine: inout [Job.ArgTemplate]) throws {
+                                                 commandLine: inout [Job.ArgTemplate],
+                                                 swiftDependencyArtifacts: inout [SwiftModuleArtifactInfo]
+  ) throws {
     // Generate a build job for the dependency module, if not already generated
     if swiftModuleBuildCache[dependencyId] == nil {
       try genSwiftModuleBuildJob(moduleId: dependencyId)
@@ -261,13 +299,17 @@ public struct ExplicitModuleBuildHandler {
     let dependencyInfo = try dependencyGraph.moduleInfo(of: dependencyId)
     let swiftModulePath = TypedVirtualPath(file: try VirtualPath(path: dependencyInfo.modulePath),
                                            type: .swiftModule)
-    commandLine.appendFlags("-swift-module-file")
-    commandLine.appendPath(swiftModulePath.file)
-    inputs.append(swiftModulePath)
+
+    // Collect the requried information about this module
+    // TODO: add .swiftdoc and .swiftsourceinfo for this module.
+    swiftDependencyArtifacts.append(
+      SwiftModuleArtifactInfo(name: dependencyId.moduleName,
+                              modulePath: swiftModulePath.file.description))
 
     // Process all transitive dependencies as direct
-    try addModuleDependencies(moduleId: dependencyId, pcmArgs: pcmArgs, inputs: &inputs,
-                              commandLine: &commandLine)
+    try addModuleDependencies(moduleId: dependencyId, pcmArgs: pcmArgs,
+                              inputs: &inputs, commandLine: &commandLine,
+                              swiftDependencyArtifacts: &swiftDependencyArtifacts)
   }
 
   /// Add a specific Clang module dependency as an input and a corresponding command
@@ -278,7 +320,9 @@ public struct ExplicitModuleBuildHandler {
                                                  dependencyId: ModuleDependencyId,
                                                  pcmArgs: [String],
                                                  inputs: inout [TypedVirtualPath],
-                                                 commandLine: inout [Job.ArgTemplate]) throws {
+                                                 commandLine: inout [Job.ArgTemplate],
+                                                 swiftDependencyArtifacts: inout [SwiftModuleArtifactInfo]
+  ) throws {
     // Generate a build job for the dependency module at the given target, if not already generated
     if clangTargetModuleBuildCache[(dependencyId, pcmArgs)] == nil {
       try genClangModuleBuildJob(moduleId: dependencyId, pcmArgs: pcmArgs)
@@ -302,8 +346,9 @@ public struct ExplicitModuleBuildHandler {
     inputs.append(clangModuleMapPath)
 
     // Process all transitive dependencies as direct
-    try addModuleDependencies(moduleId: dependencyId, pcmArgs: pcmArgs, inputs: &inputs,
-                              commandLine: &commandLine)
+    try addModuleDependencies(moduleId: dependencyId, pcmArgs: pcmArgs,
+                              inputs: &inputs, commandLine: &commandLine,
+                              swiftDependencyArtifacts: &swiftDependencyArtifacts)
   }
 }
 
diff --git a/Sources/SwiftDriver/Explicit Module Builds/InterModuleDependencyGraph.swift b/Sources/SwiftDriver/Explicit Module Builds/InterModuleDependencyGraph.swift
@@ -1,4 +1,4 @@
-//===--------------- ModuleDependencyGraph.swift --------------------------===//
+//===--------------- InterModuleDependencyGraph.swift ---------------------===//
 //
 // This source file is part of the Swift.org open source project
 //
diff --git a/Sources/SwiftDriver/Explicit Module Builds/SwiftModuleArtifacts.swift b/Sources/SwiftDriver/Explicit Module Builds/SwiftModuleArtifacts.swift
@@ -0,0 +1,35 @@
+//===--------------- SwiftModuleArtifacts.swift ---------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+import Foundation
+
+/// Describes a given Swift module's pre-built module artifacts:
+/// - Swift Module (name)
+/// - Swift Module Path
+/// - Swift Doc Path
+/// - Swift Source Info Path
+public struct SwiftModuleArtifactInfo: Codable {
+  /// The module's name
+  public let moduleName: String
+  /// The path for the module's .swiftmodule file
+  public let modulePath: String
+  /// The path for the module's .swiftdoc file
+  public let docPath: String?
+  /// The path for the module's .swiftsourceinfo file
+  public let sourceInfoPath: String?
+
+  init(name: String, modulePath: String, docPath: String? = nil, sourceInfoPath: String? = nil) {
+    self.moduleName = name
+    self.modulePath = modulePath
+    self.docPath = docPath
+    self.sourceInfoPath = sourceInfoPath
+  }
+}
diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift
@@ -69,7 +69,7 @@ extension Driver {
          .swiftDocumentation, .swiftInterface,
          .swiftSourceInfoFile, .raw_sib, .llvmBitcode, .diagnostics,
          .objcHeader, .swiftDeps, .remap, .importedModules, .tbd, .moduleTrace,
-         .indexData, .optimizationRecord, .pcm, .pch, .clangModuleMap, nil:
+         .indexData, .optimizationRecord, .pcm, .pch, .clangModuleMap, .jsonSwiftArtifacts, nil:
       return false
     }
   }
@@ -254,7 +254,8 @@ extension FileType {
 
     case .swift, .dSYM, .autolink, .dependencies, .swiftDocumentation, .pcm,
          .diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd,
-         .optimizationRecord, .swiftInterface, .swiftSourceInfoFile, .clangModuleMap:
+         .optimizationRecord, .swiftInterface, .swiftSourceInfoFile, .clangModuleMap,
+         .jsonSwiftArtifacts:
       fatalError("Output type can never be a primary output")
     }
   }
diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift
@@ -227,7 +227,8 @@ extension Driver {
       fatalError("Attempting to perform Explicit Module Build job generation, but the Inter Module Dependency Graph does not exist.")
     }
     explicitModuleBuildHandler = try ExplicitModuleBuildHandler(dependencyGraph: dependencyGraph,
-                                                                toolchain: toolchain)
+                                                                toolchain: toolchain,
+                                                                fileSystem: fileSystem)
     return try explicitModuleBuildHandler!.generateExplicitModuleDependenciesBuildJobs()
   }
 
diff --git a/Sources/SwiftDriver/Utilities/FileType.swift b/Sources/SwiftDriver/Utilities/FileType.swift
@@ -90,6 +90,9 @@ public enum FileType: String, Hashable, CaseIterable, Codable {
   /// JSON-based Module Dependency Scanner output
   case jsonDependencies = "dependencies.json"
 
+  /// JSON-based binary Swift module artifact description
+  case jsonSwiftArtifacts = "artifacts.json"
+
   /// Module trace file.
   ///
   /// Module traces are used by Apple's internal build infrastructure. Apple
@@ -147,6 +150,9 @@ extension FileType: CustomStringConvertible {
     case .jsonDependencies:
       return "json-dependencies"
 
+    case .jsonSwiftArtifacts:
+      return "json-module-artifacts"
+
     case .importedModules:
       return "imported-modules"
 
@@ -176,7 +182,7 @@ extension FileType {
          .importedModules, .indexData, .remap, .dSYM, .autolink, .dependencies,
          .swiftDocumentation, .pcm, .diagnostics, .objcHeader, .image,
          .swiftDeps, .moduleTrace, .tbd, .optimizationRecord, .swiftInterface,
-         .swiftSourceInfoFile, .jsonDependencies, .clangModuleMap:
+         .swiftSourceInfoFile, .jsonDependencies, .jsonSwiftArtifacts, .clangModuleMap:
       return false
     }
   }
@@ -247,6 +253,8 @@ extension FileType {
       return "swift-dependencies"
     case .jsonDependencies:
       return "json-dependencies"
+    case .jsonSwiftArtifacts:
+      return "json-module-artifacts"
     case .importedModules:
       return "imported-modules"
     case .moduleTrace:
@@ -266,7 +274,8 @@ extension FileType {
     switch self {
     case .swift, .sil, .dependencies, .assembly, .ast, .raw_sil, .llvmIR,
          .objcHeader, .autolink, .importedModules, .tbd, .moduleTrace,
-         .optimizationRecord, .swiftInterface, .jsonDependencies, .clangModuleMap:
+         .optimizationRecord, .swiftInterface, .jsonDependencies, .jsonSwiftArtifacts,
+         .clangModuleMap:
       return true
     case .image, .object, .dSYM, .pch, .sib, .raw_sib, .swiftModule,
          .swiftDocumentation, .swiftSourceInfoFile, .llvmBitcode, .diagnostics,
@@ -285,7 +294,7 @@ extension FileType {
          .swiftModule, .swiftDocumentation, .swiftInterface, .swiftSourceInfoFile,
          .raw_sil, .raw_sib, .diagnostics, .objcHeader, .swiftDeps, .remap, .importedModules,
          .tbd, .moduleTrace, .indexData, .optimizationRecord, .pcm, .pch, .jsonDependencies,
-         .clangModuleMap:
+         .jsonSwiftArtifacts, .clangModuleMap:
       return false
     }
   }
diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift
@@ -59,14 +59,23 @@ private func checkExplicitModuleBuildJobDependencies(job: Job,
     let dependencyInfo = moduleDependencyGraph.modules[dependencyId]!
     switch dependencyInfo.details {
       case .swift:
-        let swiftDependencyModulePath =
-          TypedVirtualPath(file: try VirtualPath(path: dependencyInfo.modulePath),
-                           type: .swiftModule)
-        XCTAssertTrue(job.inputs.contains(swiftDependencyModulePath))
-        XCTAssertTrue(job.commandLine.contains(
-                        .flag(String("-swift-module-file"))))
-        XCTAssertTrue(
-          job.commandLine.contains(.path(try VirtualPath(path: dependencyInfo.modulePath))))
+        // Load the dependency JSON and verify this dependency was encoded correctly
+        let explicitDepsFlag =
+          SwiftDriver.Job.ArgTemplate.flag(String("-explicit-swift-module-map-file"))
+        XCTAssert(job.commandLine.contains(explicitDepsFlag))
+        let jsonDepsPathIndex = job.commandLine.firstIndex(of: explicitDepsFlag)
+        let jsonDepsPathArg = job.commandLine[jsonDepsPathIndex! + 1]
+        guard case .path(let jsonDepsPath) = jsonDepsPathArg else {
+          XCTFail("No JSON dependency file path found.")
+          return
+        }
+        let contents =
+          try localFileSystem.readFileContents(jsonDepsPath.absolutePath!)
+        let dependencyInfoList = try JSONDecoder().decode(Array<SwiftModuleArtifactInfo>.self,
+                                                      from: Data(contents.contents))
+        let dependencyArtifacts =
+          dependencyInfoList.first(where:{ $0.moduleName == dependencyId.moduleName })
+        XCTAssertEqual(dependencyArtifacts!.modulePath, dependencyInfo.modulePath)
       case .clang(let clangDependencyDetails):
         let clangDependencyModulePathString =
           try ExplicitModuleBuildHandler.targetEncodedClangModuleFilePath(
@@ -113,7 +122,8 @@ final class ExplicitModuleBuildTests: XCTestCase {
               InterModuleDependencyGraph.self,
               from: ModuleDependenciesInputs.fastDependencyScannerOutput.data(using: .utf8)!)
       driver.explicitModuleBuildHandler = try ExplicitModuleBuildHandler(dependencyGraph: moduleDependencyGraph,
-                                                                         toolchain: driver.toolchain)
+                                                                         toolchain: driver.toolchain,
+                                                                         fileSystem: localFileSystem)
       let modulePrebuildJobs =
         try driver.explicitModuleBuildHandler!.generateExplicitModuleDependenciesBuildJobs()
       XCTAssertEqual(modulePrebuildJobs.count, 4)
@@ -223,4 +233,34 @@ final class ExplicitModuleBuildTests: XCTestCase {
       }
     }
   }
+
+  func testExplicitSwiftModuleMap() throws {
+    let jsonExample : String = """
+    [
+      {
+        "SwiftModule": "A",
+        "SwiftModulePath": "A.swiftmodule",
+        "SwiftDocPath": "A.swiftdoc",
+        "SwiftSourceInfoPath": "A.swiftsourceinfo"
+      },
+      {
+        "SwiftModule": "B",
+        "SwiftModulePath": "B.swiftmodule",
+        "SwiftDocPath": "B.swiftdoc",
+        "SwiftSourceInfoPath": "B.swiftsourceinfo"
+      }
+    ]
+    """
+    let moduleMap = try JSONDecoder().decode(Array<SwiftModuleArtifactInfo>.self,
+                                             from: jsonExample.data(using: .utf8)!)
+    XCTAssertEqual(moduleMap.count, 2)
+    XCTAssertEqual(moduleMap[0].moduleName, "A")
+    XCTAssertEqual(moduleMap[0].modulePath, "A.swiftmodule")
+    XCTAssertEqual(moduleMap[0].docPath, "A.swiftdoc")
+    XCTAssertEqual(moduleMap[0].sourceInfoPath, "A.swiftsourceinfo")
+    XCTAssertEqual(moduleMap[1].moduleName, "B")
+    XCTAssertEqual(moduleMap[1].modulePath, "B.swiftmodule")
+    XCTAssertEqual(moduleMap[1].docPath, "B.swiftdoc")
+    XCTAssertEqual(moduleMap[1].sourceInfoPath, "B.swiftsourceinfo")
+  }
 }

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-//===--------------- ModuleDependencyGraph.swift --------------------------===//`
	`1`	`+//===--------------- InterModuleDependencyGraph.swift ---------------------===//`
`2`	`2`	`//`
`3`	`3`	`// This source file is part of the Swift.org open source project`
`4`	`4`	`//`
Original file line number	Diff line number	Diff line change
`@@ -69,7 +69,7 @@ extension Driver {`
`69`	`69`	`.swiftDocumentation, .swiftInterface,`
`70`	`70`	`.swiftSourceInfoFile, .raw_sib, .llvmBitcode, .diagnostics,`
`71`	`71`	`.objcHeader, .swiftDeps, .remap, .importedModules, .tbd, .moduleTrace,`
`72`		`- .indexData, .optimizationRecord, .pcm, .pch, .clangModuleMap, nil:`
	`72`	`+ .indexData, .optimizationRecord, .pcm, .pch, .clangModuleMap, .jsonSwiftArtifacts, nil:`
`73`	`73`	`return false`
`74`	`74`	`}`
`75`	`75`	`}`
`@@ -254,7 +254,8 @@ extension FileType {`
`254`	`254`
`255`	`255`	`case .swift, .dSYM, .autolink, .dependencies, .swiftDocumentation, .pcm,`
`256`	`256`	`.diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd,`
`257`		`- .optimizationRecord, .swiftInterface, .swiftSourceInfoFile, .clangModuleMap:`
	`257`	`+ .optimizationRecord, .swiftInterface, .swiftSourceInfoFile, .clangModuleMap,`
	`258`	`+ .jsonSwiftArtifacts:`
`258`	`259`	`fatalError("Output type can never be a primary output")`
`259`	`260`	`}`
`260`	`261`	`}`
Original file line number	Diff line number	Diff line change
`@@ -227,7 +227,8 @@ extension Driver {`
`227`	`227`	`fatalError("Attempting to perform Explicit Module Build job generation, but the Inter Module Dependency Graph does not exist.")`
`228`	`228`	`}`
`229`	`229`	`explicitModuleBuildHandler = try ExplicitModuleBuildHandler(dependencyGraph: dependencyGraph,`
`230`		`- toolchain: toolchain)`
	`230`	`+ toolchain: toolchain,`
	`231`	`+ fileSystem: fileSystem)`
`231`	`232`	`return try explicitModuleBuildHandler!.generateExplicitModuleDependenciesBuildJobs()`
`232`	`233`	`}`
`233`	`234`