[Explicit Module Builds] Emit a JSON file that specifies details of explicit Swift module dependencies

This PR implements the interface expected by the frontend, as built in:
swiftlang/swift#32355
swiftlang/swift#32450

Swift module's explicit Swift module dependencies are now encoded in a JSON file, passed with `-explicit-swift-module-map-file`

Resolves rdar://problem/64533451

Author: artemcm

Sources/SwiftDriver/CMakeLists.txt

@@ -11,6 +11,7 @@ add_library(SwiftDriver
   "Explicit Module Builds/ExplicitModuleBuildHandler.swift"   
   "Explicit Module Builds/InterModuleDependencyGraph.swift"
   "Explicit Module Builds/ModuleDependencyScanning.swift"
+  "Explicit Module Builds/SwiftModuleArtifacts.swift"
 
   Driver/CompilerMode.swift
   Driver/DebugInfo.swift

Sources/SwiftDriver/Explicit Module Builds/ExplicitModuleBuildHandler.swift

@@ -31,9 +31,23 @@ public struct ExplicitModuleBuildHandler {
   /// The toolchain to be used for frontend job generation.
   private let toolchain: Toolchain
 
-  public init(dependencyGraph: InterModuleDependencyGraph, toolchain: Toolchain) throws {
+  /// The file system which we should interact with.
+  private let fileSystem: FileSystem
+
+  /// Path to the directory that will contain the temporary files.
+  /// e.g. Explicit Swift module artifact files
+  private let temporaryDirectory: AbsolutePath
+
+  public init(dependencyGraph: InterModuleDependencyGraph, toolchain: Toolchain,
+              fileSystem: FileSystem) throws {
     self.dependencyGraph = dependencyGraph
     self.toolchain = toolchain
+    self.fileSystem = fileSystem
+    self.temporaryDirectory = try withTemporaryDirectory(removeTreeOnDeinit: false) { path in
+      // FIXME: TSC removes empty directories even when removeTreeOnDeinit is false. This seems like a bug.
+      try fileSystem.writeFileContents(path.appending(component: ".keep-directory")) { $0 <<< "" }
+      return path
+    }
   }
 
   /// Generate build jobs for all dependencies of the main module.
@@ -207,6 +221,19 @@ public struct ExplicitModuleBuildHandler {
     )
   }
 
+  /// Store the output file artifacts for a given module in a JSON file, return the file's path.
+  private func serializeModuleDependencies(for moduleId: ModuleDependencyId,
+                                           dependencyArtifacts: [SwiftModuleArtifactInfo]
+  ) throws -> AbsolutePath {
+    let dependencyFilePath =
+      temporaryDirectory.appending(component: "\(moduleId.moduleName)-dependencies.json")
+    let encoder = JSONEncoder()
+    encoder.outputFormatting = [.prettyPrinted]
+    let contents = try encoder.encode(dependencyArtifacts)
+    try fileSystem.writeFileContents(dependencyFilePath, bytes: ByteString(contents))
+    return dependencyFilePath
+  }
+
   /// For the specified module, update the given command line flags and inputs
   /// to use explicitly-built module dependencies.
   /// 
@@ -217,27 +244,43 @@ public struct ExplicitModuleBuildHandler {
                                                           inputs: inout [TypedVirtualPath],
                                                           commandLine: inout [Job.ArgTemplate]
   ) throws {
+
     // Prohibit the frontend from implicitly building textual modules into binary modules.
     commandLine.appendFlags("-disable-implicit-swift-modules", "-Xcc", "-Xclang", "-Xcc",
                             "-fno-implicit-modules")
+    var swiftDependencyArtifacts: [SwiftModuleArtifactInfo] = []
     try addModuleDependencies(moduleId: moduleId, pcmArgs: pcmArgs, inputs: &inputs,
-                              commandLine: &commandLine)
+                              commandLine: &commandLine,
+                              swiftDependencyArtifacts: &swiftDependencyArtifacts)
+
+    if (swiftDependencyArtifacts.isEmpty) {
+      let dependencyFile = try serializeModuleDependencies(for: moduleId,
+                                                           dependencyArtifacts: swiftDependencyArtifacts)
+      commandLine.appendFlag("-explicit-swift-module-map-file")
+      commandLine.appendPath(dependencyFile)
+      inputs.append(TypedVirtualPath(file: try VirtualPath(path: dependencyFile.pathString),
+                                     type: .jsonSwiftArtifacts))
+    }
   }
 
   /// Add a specific module dependency as an input and a corresponding command
   /// line flag. Dispatches to clang and swift-specific variants.
   mutating private func addModuleDependencies(moduleId: ModuleDependencyId,
                                               pcmArgs: [String],
                                               inputs: inout [TypedVirtualPath],
-                                              commandLine: inout [Job.ArgTemplate]) throws {
+                                              commandLine: inout [Job.ArgTemplate],
+                                              swiftDependencyArtifacts: inout [SwiftModuleArtifactInfo]
+  ) throws {
     for dependencyId in try dependencyGraph.moduleInfo(of: moduleId).directDependencies {
       switch dependencyId {
         case .swift:
           try addSwiftModuleDependency(moduleId: moduleId, dependencyId: dependencyId,
-                                       pcmArgs: pcmArgs, inputs: &inputs, commandLine: &commandLine)
+                                       pcmArgs: pcmArgs, inputs: &inputs, commandLine: &commandLine,
+                                       swiftDependencyArtifacts: &swiftDependencyArtifacts)
         case .clang:
           try addClangModuleDependency(moduleId: moduleId, dependencyId: dependencyId,
-                                       pcmArgs: pcmArgs, inputs: &inputs, commandLine: &commandLine)
+                                       pcmArgs: pcmArgs, inputs: &inputs, commandLine: &commandLine,
+                                       swiftDependencyArtifacts: &swiftDependencyArtifacts)
       }
     }
   }
@@ -250,7 +293,9 @@ public struct ExplicitModuleBuildHandler {
                                                  dependencyId: ModuleDependencyId,
                                                  pcmArgs: [String],
                                                  inputs: inout [TypedVirtualPath],
-                                                 commandLine: inout [Job.ArgTemplate]) throws {
+                                                 commandLine: inout [Job.ArgTemplate],
+                                                 swiftDependencyArtifacts: inout [SwiftModuleArtifactInfo]
+  ) throws {
     // Generate a build job for the dependency module, if not already generated
     if swiftModuleBuildCache[dependencyId] == nil {
       try genSwiftModuleBuildJob(moduleId: dependencyId)
@@ -261,13 +306,17 @@ public struct ExplicitModuleBuildHandler {
     let dependencyInfo = try dependencyGraph.moduleInfo(of: dependencyId)
     let swiftModulePath = TypedVirtualPath(file: try VirtualPath(path: dependencyInfo.modulePath),
                                            type: .swiftModule)
-    commandLine.appendFlags("-swift-module-file")
-    commandLine.appendPath(swiftModulePath.file)
-    inputs.append(swiftModulePath)
+
+    // Collect the requried information about this module
+    // TODO: add .swiftdoc and .swiftsourceinfo for this module.
+    swiftDependencyArtifacts.append(
+      SwiftModuleArtifactInfo(name: dependencyId.moduleName,
+                              modulePath: swiftModulePath.file.description))
 
     // Process all transitive dependencies as direct
-    try addModuleDependencies(moduleId: dependencyId, pcmArgs: pcmArgs, inputs: &inputs,
-                              commandLine: &commandLine)
+    try addModuleDependencies(moduleId: dependencyId, pcmArgs: pcmArgs,
+                              inputs: &inputs, commandLine: &commandLine,
+                              swiftDependencyArtifacts: &swiftDependencyArtifacts)
   }
 
   /// Add a specific Clang module dependency as an input and a corresponding command
@@ -278,7 +327,9 @@ public struct ExplicitModuleBuildHandler {
                                                  dependencyId: ModuleDependencyId,
                                                  pcmArgs: [String],
                                                  inputs: inout [TypedVirtualPath],
-                                                 commandLine: inout [Job.ArgTemplate]) throws {
+                                                 commandLine: inout [Job.ArgTemplate],
+                                                 swiftDependencyArtifacts: inout [SwiftModuleArtifactInfo]
+  ) throws {
     // Generate a build job for the dependency module at the given target, if not already generated
     if clangTargetModuleBuildCache[(dependencyId, pcmArgs)] == nil {
       try genClangModuleBuildJob(moduleId: dependencyId, pcmArgs: pcmArgs)
@@ -302,8 +353,9 @@ public struct ExplicitModuleBuildHandler {
     inputs.append(clangModuleMapPath)
 
     // Process all transitive dependencies as direct
-    try addModuleDependencies(moduleId: dependencyId, pcmArgs: pcmArgs, inputs: &inputs,
-                              commandLine: &commandLine)
+    try addModuleDependencies(moduleId: dependencyId, pcmArgs: pcmArgs,
+                              inputs: &inputs, commandLine: &commandLine,
+                              swiftDependencyArtifacts: &swiftDependencyArtifacts)
   }
 }
 

Sources/SwiftDriver/Explicit Module Builds/InterModuleDependencyGraph.swift

@@ -1,4 +1,4 @@
-//===--------------- ModuleDependencyGraph.swift --------------------------===//
+//===--------------- InterModuleDependencyGraph.swift ---------------------===//
 //
 // This source file is part of the Swift.org open source project
 //

Sources/SwiftDriver/Explicit Module Builds/SwiftModuleArtifacts.swift

@@ -0,0 +1,42 @@
+//===--------------- SwiftModuleArtifacts.swift ---------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+import Foundation
+
+/// Describes a given Swift module's pre-built module artifacts:
+/// - Swift Module (name)
+/// - Swift Module Path
+/// - Swift Doc Path
+/// - Swift Source Info Path
+public struct SwiftModuleArtifactInfo: Codable {
+  /// The module's name
+  public let moduleName: String
+  /// The path for the module's .swiftmodule file
+  public let modulePath: String
+  /// The path for the module's .swiftdoc file
+  public let docPath: String?
+  /// The path for the module's .swiftsourceinfo file
+  public let sourceInfoPath: String?
+
+  init(name: String, modulePath: String, docPath: String? = nil, sourceInfoPath: String? = nil) {
+    self.moduleName = name
+    self.modulePath = modulePath
+    self.docPath = docPath
+    self.sourceInfoPath = sourceInfoPath
+  }
+
+  private enum CodingKeys : String, CodingKey {
+    case moduleName = "SwiftModule"
+    case modulePath = "SwiftModulePath"
+    case docPath = "SwiftDocPath"
+    case sourceInfoPath = "SwiftSourceInfoPath"
+  }
+}

Sources/SwiftDriver/Jobs/CompileJob.swift

@@ -69,7 +69,7 @@ extension Driver {
          .swiftDocumentation, .swiftInterface,
          .swiftSourceInfoFile, .raw_sib, .llvmBitcode, .diagnostics,
          .objcHeader, .swiftDeps, .remap, .importedModules, .tbd, .moduleTrace,
-         .indexData, .optimizationRecord, .pcm, .pch, .clangModuleMap, nil:
+         .indexData, .optimizationRecord, .pcm, .pch, .clangModuleMap, .jsonSwiftArtifacts, nil:
       return false
     }
   }
@@ -254,7 +254,8 @@ extension FileType {
 
     case .swift, .dSYM, .autolink, .dependencies, .swiftDocumentation, .pcm,
          .diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd,
-         .optimizationRecord, .swiftInterface, .swiftSourceInfoFile, .clangModuleMap:
+         .optimizationRecord, .swiftInterface, .swiftSourceInfoFile, .clangModuleMap,
+         .jsonSwiftArtifacts:
       fatalError("Output type can never be a primary output")
     }
   }

Sources/SwiftDriver/Jobs/Planning.swift

@@ -227,7 +227,8 @@ extension Driver {
       fatalError("Attempting to perform Explicit Module Build job generation, but the Inter Module Dependency Graph does not exist.")
     }
     explicitModuleBuildHandler = try ExplicitModuleBuildHandler(dependencyGraph: dependencyGraph,
-                                                                toolchain: toolchain)
+                                                                toolchain: toolchain,
+                                                                fileSystem: fileSystem)
     return try explicitModuleBuildHandler!.generateExplicitModuleDependenciesBuildJobs()
   }
 

Sources/SwiftDriver/Utilities/FileType.swift

@@ -90,6 +90,9 @@ public enum FileType: String, Hashable, CaseIterable, Codable {
   /// JSON-based Module Dependency Scanner output
   case jsonDependencies = "dependencies.json"
 
+  /// JSON-based binary Swift module artifact description
+  case jsonSwiftArtifacts = "artifacts.json"
+
   /// Module trace file.
   ///
   /// Module traces are used by Apple's internal build infrastructure. Apple
@@ -147,6 +150,9 @@ extension FileType: CustomStringConvertible {
     case .jsonDependencies:
       return "json-dependencies"
 
+    case .jsonSwiftArtifacts:
+      return "json-module-artifacts"
+
     case .importedModules:
       return "imported-modules"
 
@@ -176,7 +182,7 @@ extension FileType {
          .importedModules, .indexData, .remap, .dSYM, .autolink, .dependencies,
          .swiftDocumentation, .pcm, .diagnostics, .objcHeader, .image,
          .swiftDeps, .moduleTrace, .tbd, .optimizationRecord, .swiftInterface,
-         .swiftSourceInfoFile, .jsonDependencies, .clangModuleMap:
+         .swiftSourceInfoFile, .jsonDependencies, .jsonSwiftArtifacts, .clangModuleMap:
       return false
     }
   }
@@ -247,6 +253,8 @@ extension FileType {
       return "swift-dependencies"
     case .jsonDependencies:
       return "json-dependencies"
+    case .jsonSwiftArtifacts:
+      return "json-module-artifacts"
     case .importedModules:
       return "imported-modules"
     case .moduleTrace:
@@ -266,7 +274,8 @@ extension FileType {
     switch self {
     case .swift, .sil, .dependencies, .assembly, .ast, .raw_sil, .llvmIR,
          .objcHeader, .autolink, .importedModules, .tbd, .moduleTrace,
-         .optimizationRecord, .swiftInterface, .jsonDependencies, .clangModuleMap:
+         .optimizationRecord, .swiftInterface, .jsonDependencies, .jsonSwiftArtifacts,
+         .clangModuleMap:
       return true
     case .image, .object, .dSYM, .pch, .sib, .raw_sib, .swiftModule,
          .swiftDocumentation, .swiftSourceInfoFile, .llvmBitcode, .diagnostics,
@@ -285,7 +294,7 @@ extension FileType {
          .swiftModule, .swiftDocumentation, .swiftInterface, .swiftSourceInfoFile,
          .raw_sil, .raw_sib, .diagnostics, .objcHeader, .swiftDeps, .remap, .importedModules,
          .tbd, .moduleTrace, .indexData, .optimizationRecord, .pcm, .pch, .jsonDependencies,
-         .clangModuleMap:
+         .jsonSwiftArtifacts, .clangModuleMap:
       return false
     }
   }

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -59,14 +59,23 @@ private func checkExplicitModuleBuildJobDependencies(job: Job,
     let dependencyInfo = moduleDependencyGraph.modules[dependencyId]!
     switch dependencyInfo.details {
       case .swift:
-        let swiftDependencyModulePath =
-          TypedVirtualPath(file: try VirtualPath(path: dependencyInfo.modulePath),
-                           type: .swiftModule)
-        XCTAssertTrue(job.inputs.contains(swiftDependencyModulePath))
-        XCTAssertTrue(job.commandLine.contains(
-                        .flag(String("-swift-module-file"))))
-        XCTAssertTrue(
-          job.commandLine.contains(.path(try VirtualPath(path: dependencyInfo.modulePath))))
+        // Load the dependency JSON and verify this dependency was encoded correctly
+        let explicitDepsFlag =
+          SwiftDriver.Job.ArgTemplate.flag(String("-explicit-swift-module-map-file"))
+        XCTAssert(job.commandLine.contains(explicitDepsFlag))
+        let jsonDepsPathIndex = job.commandLine.firstIndex(of: explicitDepsFlag)
+        let jsonDepsPathArg = job.commandLine[jsonDepsPathIndex! + 1]
+        guard case .path(let jsonDepsPath) = jsonDepsPathArg else {
+          XCTFail("No JSON dependency file path found.")
+          return
+        }
+        let contents =
+          try localFileSystem.readFileContents(jsonDepsPath.absolutePath!)
+        let dependencyInfoList = try JSONDecoder().decode(Array<SwiftModuleArtifactInfo>.self,
+                                                      from: Data(contents.contents))
+        let dependencyArtifacts =
+          dependencyInfoList.first(where:{ $0.moduleName == dependencyId.moduleName })
+        XCTAssertEqual(dependencyArtifacts!.modulePath, dependencyInfo.modulePath)
       case .clang(let clangDependencyDetails):
         let clangDependencyModulePathString =
           try ExplicitModuleBuildHandler.targetEncodedClangModuleFilePath(
@@ -113,7 +122,8 @@ final class ExplicitModuleBuildTests: XCTestCase {
               InterModuleDependencyGraph.self,
               from: ModuleDependenciesInputs.fastDependencyScannerOutput.data(using: .utf8)!)
       driver.explicitModuleBuildHandler = try ExplicitModuleBuildHandler(dependencyGraph: moduleDependencyGraph,
-                                                                         toolchain: driver.toolchain)
+                                                                         toolchain: driver.toolchain,
+                                                                         fileSystem: localFileSystem)
       let modulePrebuildJobs =
         try driver.explicitModuleBuildHandler!.generateExplicitModuleDependenciesBuildJobs()
       XCTAssertEqual(modulePrebuildJobs.count, 4)
@@ -223,4 +233,34 @@ final class ExplicitModuleBuildTests: XCTestCase {
       }
     }
   }
+
+  func testExplicitSwiftModuleMap() throws {
+    let jsonExample : String = """
+    [
+      {
+        "SwiftModule": "A",
+        "SwiftModulePath": "A.swiftmodule",
+        "SwiftDocPath": "A.swiftdoc",
+        "SwiftSourceInfoPath": "A.swiftsourceinfo"
+      },
+      {
+        "SwiftModule": "B",
+        "SwiftModulePath": "B.swiftmodule",
+        "SwiftDocPath": "B.swiftdoc",
+        "SwiftSourceInfoPath": "B.swiftsourceinfo"
+      }
+    ]
+    """
+    let moduleMap = try JSONDecoder().decode(Array<SwiftModuleArtifactInfo>.self,
+                                             from: jsonExample.data(using: .utf8)!)
+    XCTAssertEqual(moduleMap.count, 2)
+    XCTAssertEqual(moduleMap[0].moduleName, "A")
+    XCTAssertEqual(moduleMap[0].modulePath, "A.swiftmodule")
+    XCTAssertEqual(moduleMap[0].docPath, "A.swiftdoc")
+    XCTAssertEqual(moduleMap[0].sourceInfoPath, "A.swiftsourceinfo")
+    XCTAssertEqual(moduleMap[1].moduleName, "B")
+    XCTAssertEqual(moduleMap[1].modulePath, "B.swiftmodule")
+    XCTAssertEqual(moduleMap[1].docPath, "B.swiftdoc")
+    XCTAssertEqual(moduleMap[1].sourceInfoPath, "B.swiftsourceinfo")
+  }
 }