diff --git a/Sources/SwiftDriver/CMakeLists.txt b/Sources/SwiftDriver/CMakeLists.txt index bc7a114a5..8b32cee7b 100644 --- a/Sources/SwiftDriver/CMakeLists.txt +++ b/Sources/SwiftDriver/CMakeLists.txt @@ -11,6 +11,7 @@ add_library(SwiftDriver "Explicit Module Builds/ExplicitModuleBuildHandler.swift" "Explicit Module Builds/InterModuleDependencyGraph.swift" "Explicit Module Builds/ModuleDependencyScanning.swift" + "Explicit Module Builds/ModuleArtifacts.swift" Driver/CompilerMode.swift Driver/DebugInfo.swift diff --git a/Sources/SwiftDriver/Explicit Module Builds/ExplicitModuleBuildHandler.swift b/Sources/SwiftDriver/Explicit Module Builds/ExplicitModuleBuildHandler.swift index 4331c7d09..af57e99f1 100644 --- a/Sources/SwiftDriver/Explicit Module Builds/ExplicitModuleBuildHandler.swift +++ b/Sources/SwiftDriver/Explicit Module Builds/ExplicitModuleBuildHandler.swift @@ -31,9 +31,23 @@ public struct ExplicitModuleBuildHandler { /// The toolchain to be used for frontend job generation. private let toolchain: Toolchain - public init(dependencyGraph: InterModuleDependencyGraph, toolchain: Toolchain) throws { + /// The file system which we should interact with. + /// FIXME: Our end goal is to not have any direct filesystem manipulation in here, but that's dependent on getting the + /// dependency scanner/dependency job generation moved into a Job. + private let fileSystem: FileSystem + + /// Path to the directory that will contain the temporary files. + /// e.g. Explicit Swift module artifact files + /// FIXME: Our end goal is to not have any direct filesystem manipulation in here, but that's dependent on getting the + /// dependency scanner/dependency job generation moved into a Job. + private let temporaryDirectory: AbsolutePath + + public init(dependencyGraph: InterModuleDependencyGraph, toolchain: Toolchain, + fileSystem: FileSystem) throws { self.dependencyGraph = dependencyGraph self.toolchain = toolchain + self.fileSystem = fileSystem + self.temporaryDirectory = try determineTempDirectory() } /// Generate build jobs for all dependencies of the main module. @@ -97,17 +111,10 @@ public struct ExplicitModuleBuildHandler { /// - Generate Job: S1 /// mutating public func generateExplicitModuleDependenciesBuildJobs() throws -> [Job] { - for moduleId in dependencyGraph.mainModule.directDependencies { - switch moduleId { - case .swift: - try genSwiftModuleBuildJob(moduleId: moduleId) - case .clang: - try genClangModuleBuildJob(moduleId: moduleId, - pcmArgs: try dependencyGraph.swiftModulePCMArgs( - of: .swift(dependencyGraph.mainModuleName))) - } - } - + var mainModuleInputs: [TypedVirtualPath] = [] + var mainModuleCommandLine: [Job.ArgTemplate] = [] + try resolveMainModuleDependencies(inputs: &mainModuleInputs, + commandLine: &mainModuleCommandLine) return Array(swiftModuleBuildCache.values) + clangTargetModuleBuildCache.allJobs } @@ -167,7 +174,6 @@ public struct ExplicitModuleBuildHandler { mutating private func genClangModuleBuildJob(moduleId: ModuleDependencyId, pcmArgs: [String]) throws { let moduleInfo = try dependencyGraph.moduleInfo(of: moduleId) - var inputs: [TypedVirtualPath] = [] var outputs: [TypedVirtualPath] = [] var commandLine: [Job.ArgTemplate] = [] @@ -207,6 +213,19 @@ public struct ExplicitModuleBuildHandler { ) } + /// Store the output file artifacts for a given module in a JSON file, return the file's path. + private func serializeModuleDependencies(for moduleId: ModuleDependencyId, + dependencyArtifacts: [SwiftModuleArtifactInfo] + ) throws -> AbsolutePath { + let dependencyFilePath = + temporaryDirectory.appending(component: "\(moduleId.moduleName)-dependencies.json") + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted] + let contents = try encoder.encode(dependencyArtifacts) + try fileSystem.writeFileContents(dependencyFilePath, bytes: ByteString(contents)) + return dependencyFilePath + } + /// For the specified module, update the given command line flags and inputs /// to use explicitly-built module dependencies. /// @@ -220,24 +239,63 @@ public struct ExplicitModuleBuildHandler { // Prohibit the frontend from implicitly building textual modules into binary modules. commandLine.appendFlags("-disable-implicit-swift-modules", "-Xcc", "-Xclang", "-Xcc", "-fno-implicit-modules") - try addModuleDependencies(moduleId: moduleId, pcmArgs: pcmArgs, inputs: &inputs, - commandLine: &commandLine) + var swiftDependencyArtifacts: [SwiftModuleArtifactInfo] = [] + var clangDependencyArtifacts: [ClangModuleArtifactInfo] = [] + try addModuleDependencies(moduleId: moduleId, pcmArgs: pcmArgs, + clangDependencyArtifacts: &clangDependencyArtifacts, + swiftDependencyArtifacts: &swiftDependencyArtifacts) + + // Swift Module dependencies are passed encoded in a JSON file as described by + // SwiftModuleArtifactInfo + if !swiftDependencyArtifacts.isEmpty { + let dependencyFile = + try serializeModuleDependencies(for: moduleId, dependencyArtifacts: swiftDependencyArtifacts) + commandLine.appendFlag("-explicit-swift-module-map-file") + commandLine.appendPath(dependencyFile) + inputs.append(TypedVirtualPath(file: try VirtualPath(path: dependencyFile.pathString), + type: .jsonSwiftArtifacts)) + // Each individual module binary is still an "input" to ensure the build system gets the + // order correctly. + for dependencyModule in swiftDependencyArtifacts { + inputs.append(TypedVirtualPath(file: try VirtualPath(path: dependencyModule.modulePath), + type: .swiftModule)) + } + } + // Clang module depenencies are specified on the command line eplicitly + for moduleArtifactInfo in clangDependencyArtifacts { + let clangModulePath = + TypedVirtualPath(file: try VirtualPath(path: moduleArtifactInfo.modulePath), + type: .pcm) + let clangModuleMapPath = + TypedVirtualPath(file: try VirtualPath(path: moduleArtifactInfo.moduleMapPath), + type: .clangModuleMap) + commandLine.appendFlags("-Xcc", "-Xclang", "-Xcc", + "-fmodule-file=\(clangModulePath.file.description)") + commandLine.appendFlags("-Xcc", "-Xclang", "-Xcc", + "-fmodule-map-file=\(clangModuleMapPath.file.description)") + inputs.append(clangModulePath) + inputs.append(clangModuleMapPath) + } } /// Add a specific module dependency as an input and a corresponding command /// line flag. Dispatches to clang and swift-specific variants. - mutating private func addModuleDependencies(moduleId: ModuleDependencyId, - pcmArgs: [String], - inputs: inout [TypedVirtualPath], - commandLine: inout [Job.ArgTemplate]) throws { + mutating private func addModuleDependencies(moduleId: ModuleDependencyId, pcmArgs: [String], + clangDependencyArtifacts: inout [ClangModuleArtifactInfo], + swiftDependencyArtifacts: inout [SwiftModuleArtifactInfo] + ) throws { for dependencyId in try dependencyGraph.moduleInfo(of: moduleId).directDependencies { switch dependencyId { case .swift: try addSwiftModuleDependency(moduleId: moduleId, dependencyId: dependencyId, - pcmArgs: pcmArgs, inputs: &inputs, commandLine: &commandLine) + pcmArgs: pcmArgs, + clangDependencyArtifacts: &clangDependencyArtifacts, + swiftDependencyArtifacts: &swiftDependencyArtifacts) case .clang: try addClangModuleDependency(moduleId: moduleId, dependencyId: dependencyId, - pcmArgs: pcmArgs, inputs: &inputs, commandLine: &commandLine) + pcmArgs: pcmArgs, + clangDependencyArtifacts: &clangDependencyArtifacts, + swiftDependencyArtifacts: &swiftDependencyArtifacts) } } } @@ -249,8 +307,9 @@ public struct ExplicitModuleBuildHandler { mutating private func addSwiftModuleDependency(moduleId: ModuleDependencyId, dependencyId: ModuleDependencyId, pcmArgs: [String], - inputs: inout [TypedVirtualPath], - commandLine: inout [Job.ArgTemplate]) throws { + clangDependencyArtifacts: inout [ClangModuleArtifactInfo], + swiftDependencyArtifacts: inout [SwiftModuleArtifactInfo] + ) throws { // Generate a build job for the dependency module, if not already generated if swiftModuleBuildCache[dependencyId] == nil { try genSwiftModuleBuildJob(moduleId: dependencyId) @@ -261,13 +320,17 @@ public struct ExplicitModuleBuildHandler { let dependencyInfo = try dependencyGraph.moduleInfo(of: dependencyId) let swiftModulePath = TypedVirtualPath(file: try VirtualPath(path: dependencyInfo.modulePath), type: .swiftModule) - commandLine.appendFlags("-swift-module-file") - commandLine.appendPath(swiftModulePath.file) - inputs.append(swiftModulePath) + + // Collect the required information about this module + // TODO: add .swiftdoc and .swiftsourceinfo for this module. + swiftDependencyArtifacts.append( + SwiftModuleArtifactInfo(name: dependencyId.moduleName, + modulePath: swiftModulePath.file.description)) // Process all transitive dependencies as direct - try addModuleDependencies(moduleId: dependencyId, pcmArgs: pcmArgs, inputs: &inputs, - commandLine: &commandLine) + try addModuleDependencies(moduleId: dependencyId, pcmArgs: pcmArgs, + clangDependencyArtifacts: &clangDependencyArtifacts, + swiftDependencyArtifacts: &swiftDependencyArtifacts) } /// Add a specific Clang module dependency as an input and a corresponding command @@ -277,8 +340,9 @@ public struct ExplicitModuleBuildHandler { mutating private func addClangModuleDependency(moduleId: ModuleDependencyId, dependencyId: ModuleDependencyId, pcmArgs: [String], - inputs: inout [TypedVirtualPath], - commandLine: inout [Job.ArgTemplate]) throws { + clangDependencyArtifacts: inout [ClangModuleArtifactInfo], + swiftDependencyArtifacts: inout [SwiftModuleArtifactInfo] + ) throws { // Generate a build job for the dependency module at the given target, if not already generated if clangTargetModuleBuildCache[(dependencyId, pcmArgs)] == nil { try genClangModuleBuildJob(moduleId: dependencyId, pcmArgs: pcmArgs) @@ -289,21 +353,18 @@ public struct ExplicitModuleBuildHandler { let dependencyInfo = try dependencyGraph.moduleInfo(of: dependencyId) let dependencyClangModuleDetails = try dependencyGraph.clangModuleDetails(of: dependencyId) let clangModulePath = - TypedVirtualPath(file: try ExplicitModuleBuildHandler.targetEncodedClangModuleFilePath( - for: dependencyInfo, pcmArgs: pcmArgs), type: .pcm) - let clangModuleMapPath = - TypedVirtualPath(file: try VirtualPath(path: dependencyClangModuleDetails.moduleMapPath), - type: .clangModuleMap) - commandLine.appendFlags("-Xcc", "-Xclang", "-Xcc", - "-fmodule-map-file=\(clangModuleMapPath.file.description)") - commandLine.appendFlags("-Xcc", "-Xclang", "-Xcc", - "-fmodule-file=\(clangModulePath.file.description)") - inputs.append(clangModulePath) - inputs.append(clangModuleMapPath) + try ExplicitModuleBuildHandler.targetEncodedClangModuleFilePath(for: dependencyInfo, + pcmArgs: pcmArgs) + + // Collect the requried information about this module + clangDependencyArtifacts.append( + ClangModuleArtifactInfo(name: dependencyId.moduleName, modulePath: clangModulePath.description, + moduleMapPath: dependencyClangModuleDetails.moduleMapPath)) // Process all transitive dependencies as direct - try addModuleDependencies(moduleId: dependencyId, pcmArgs: pcmArgs, inputs: &inputs, - commandLine: &commandLine) + try addModuleDependencies(moduleId: dependencyId, pcmArgs: pcmArgs, + clangDependencyArtifacts: &clangDependencyArtifacts, + swiftDependencyArtifacts: &swiftDependencyArtifacts) } } diff --git a/Sources/SwiftDriver/Explicit Module Builds/InterModuleDependencyGraph.swift b/Sources/SwiftDriver/Explicit Module Builds/InterModuleDependencyGraph.swift index 806022d0a..701e89f6f 100644 --- a/Sources/SwiftDriver/Explicit Module Builds/InterModuleDependencyGraph.swift +++ b/Sources/SwiftDriver/Explicit Module Builds/InterModuleDependencyGraph.swift @@ -1,4 +1,4 @@ -//===--------------- ModuleDependencyGraph.swift --------------------------===// +//===--------------- InterModuleDependencyGraph.swift ---------------------===// // // This source file is part of the Swift.org open source project // diff --git a/Sources/SwiftDriver/Explicit Module Builds/ModuleArtifacts.swift b/Sources/SwiftDriver/Explicit Module Builds/ModuleArtifacts.swift new file mode 100644 index 000000000..bd0dce62a --- /dev/null +++ b/Sources/SwiftDriver/Explicit Module Builds/ModuleArtifacts.swift @@ -0,0 +1,54 @@ +//===--------------- SwiftModuleArtifacts.swift ---------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +import Foundation + +/// Describes a given Swift module's pre-built module artifacts: +/// - Swift Module (name) +/// - Swift Module Path +/// - Swift Doc Path +/// - Swift Source Info Path +public struct SwiftModuleArtifactInfo: Codable { + /// The module's name + public let moduleName: String + /// The path for the module's .swiftmodule file + public let modulePath: String + /// The path for the module's .swiftdoc file + public let docPath: String? + /// The path for the module's .swiftsourceinfo file + public let sourceInfoPath: String? + + init(name: String, modulePath: String, docPath: String? = nil, sourceInfoPath: String? = nil) { + self.moduleName = name + self.modulePath = modulePath + self.docPath = docPath + self.sourceInfoPath = sourceInfoPath + } +} + +/// Describes a given Clang module's pre-built module artifacts: +/// - Clang Module (name) +/// - Clang Module (PCM) Path +/// - Clang Module Map Path +public struct ClangModuleArtifactInfo { + /// The module's name + public let moduleName: String + /// The path for the module's .pcm file + public let modulePath: String + /// The path for this module's .modulemap file + public let moduleMapPath: String + + init(name: String, modulePath: String, moduleMapPath: String) { + self.moduleName = name + self.modulePath = modulePath + self.moduleMapPath = moduleMapPath + } +} diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift index 18599b6a6..3768d2bdf 100644 --- a/Sources/SwiftDriver/Jobs/CompileJob.swift +++ b/Sources/SwiftDriver/Jobs/CompileJob.swift @@ -69,7 +69,9 @@ extension Driver { .swiftDocumentation, .swiftInterface, .swiftSourceInfoFile, .raw_sib, .llvmBitcode, .diagnostics, .objcHeader, .swiftDeps, .remap, .importedModules, .tbd, .moduleTrace, - .indexData, .optimizationRecord, .pcm, .pch, .clangModuleMap, .jsonTargetInfo, nil: + + .indexData, .optimizationRecord, .pcm, .pch, .clangModuleMap, + .jsonTargetInfo, .jsonSwiftArtifacts, nil: return false } } @@ -256,7 +258,8 @@ extension FileType { case .swift, .dSYM, .autolink, .dependencies, .swiftDocumentation, .pcm, .diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd, - .optimizationRecord, .swiftInterface, .swiftSourceInfoFile, .clangModuleMap: + .optimizationRecord, .swiftInterface, .swiftSourceInfoFile, .clangModuleMap, + .jsonSwiftArtifacts: fatalError("Output type can never be a primary output") } } diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift index 46adf61f0..a941e5169 100644 --- a/Sources/SwiftDriver/Jobs/Planning.swift +++ b/Sources/SwiftDriver/Jobs/Planning.swift @@ -232,7 +232,8 @@ extension Driver { recordedInputModificationDates: recordedInputModificationDates) explicitModuleBuildHandler = try ExplicitModuleBuildHandler(dependencyGraph: dependencyGraph, - toolchain: toolchain) + toolchain: toolchain, + fileSystem: fileSystem) return try explicitModuleBuildHandler!.generateExplicitModuleDependenciesBuildJobs() } diff --git a/Sources/SwiftDriver/Utilities/FileType.swift b/Sources/SwiftDriver/Utilities/FileType.swift index c2728411e..8150503fc 100644 --- a/Sources/SwiftDriver/Utilities/FileType.swift +++ b/Sources/SwiftDriver/Utilities/FileType.swift @@ -93,6 +93,9 @@ public enum FileType: String, Hashable, CaseIterable, Codable { /// JSON-based -print-target-info output case jsonTargetInfo = "targetInfo.json" + /// JSON-based binary Swift module artifact description + case jsonSwiftArtifacts = "artifacts.json" + /// Module trace file. /// /// Module traces are used by Apple's internal build infrastructure. Apple @@ -153,6 +156,9 @@ extension FileType: CustomStringConvertible { case .jsonTargetInfo: return "json-target-info" + case .jsonSwiftArtifacts: + return "json-module-artifacts" + case .importedModules: return "imported-modules" @@ -182,7 +188,8 @@ extension FileType { .importedModules, .indexData, .remap, .dSYM, .autolink, .dependencies, .swiftDocumentation, .pcm, .diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd, .optimizationRecord, .swiftInterface, - .swiftSourceInfoFile, .jsonDependencies, .clangModuleMap, .jsonTargetInfo: + .swiftSourceInfoFile, .jsonDependencies, .clangModuleMap, .jsonTargetInfo, + .jsonSwiftArtifacts: return false } } @@ -255,6 +262,8 @@ extension FileType { return "json-dependencies" case .jsonTargetInfo: return "json-target-info" + case .jsonSwiftArtifacts: + return "json-module-artifacts" case .importedModules: return "imported-modules" case .moduleTrace: @@ -274,7 +283,8 @@ extension FileType { switch self { case .swift, .sil, .dependencies, .assembly, .ast, .raw_sil, .llvmIR, .objcHeader, .autolink, .importedModules, .tbd, .moduleTrace, - .optimizationRecord, .swiftInterface, .jsonDependencies, .clangModuleMap, .jsonTargetInfo: + .optimizationRecord, .swiftInterface, .jsonDependencies, .clangModuleMap, .jsonTargetInfo, + .jsonSwiftArtifacts: return true case .image, .object, .dSYM, .pch, .sib, .raw_sib, .swiftModule, .swiftDocumentation, .swiftSourceInfoFile, .llvmBitcode, .diagnostics, @@ -293,7 +303,7 @@ extension FileType { .swiftModule, .swiftDocumentation, .swiftInterface, .swiftSourceInfoFile, .raw_sil, .raw_sib, .diagnostics, .objcHeader, .swiftDeps, .remap, .importedModules, .tbd, .moduleTrace, .indexData, .optimizationRecord, .pcm, .pch, .jsonDependencies, - .clangModuleMap, .jsonTargetInfo: + .clangModuleMap, .jsonTargetInfo, .jsonSwiftArtifacts: return false } } diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index c94ec0557..e314895ae 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -59,14 +59,23 @@ private func checkExplicitModuleBuildJobDependencies(job: Job, let dependencyInfo = moduleDependencyGraph.modules[dependencyId]! switch dependencyInfo.details { case .swift: - let swiftDependencyModulePath = - TypedVirtualPath(file: try VirtualPath(path: dependencyInfo.modulePath), - type: .swiftModule) - XCTAssertTrue(job.inputs.contains(swiftDependencyModulePath)) - XCTAssertTrue(job.commandLine.contains( - .flag(String("-swift-module-file")))) - XCTAssertTrue( - job.commandLine.contains(.path(try VirtualPath(path: dependencyInfo.modulePath)))) + // Load the dependency JSON and verify this dependency was encoded correctly + let explicitDepsFlag = + SwiftDriver.Job.ArgTemplate.flag(String("-explicit-swift-module-map-file")) + XCTAssert(job.commandLine.contains(explicitDepsFlag)) + let jsonDepsPathIndex = job.commandLine.firstIndex(of: explicitDepsFlag) + let jsonDepsPathArg = job.commandLine[jsonDepsPathIndex! + 1] + guard case .path(let jsonDepsPath) = jsonDepsPathArg else { + XCTFail("No JSON dependency file path found.") + return + } + let contents = + try localFileSystem.readFileContents(jsonDepsPath.absolutePath!) + let dependencyInfoList = try JSONDecoder().decode(Array.self, + from: Data(contents.contents)) + let dependencyArtifacts = + dependencyInfoList.first(where:{ $0.moduleName == dependencyId.moduleName }) + XCTAssertEqual(dependencyArtifacts!.modulePath, dependencyInfo.modulePath) case .clang(let clangDependencyDetails): let clangDependencyModulePathString = try ExplicitModuleBuildHandler.targetEncodedClangModuleFilePath( @@ -113,7 +122,8 @@ final class ExplicitModuleBuildTests: XCTestCase { InterModuleDependencyGraph.self, from: ModuleDependenciesInputs.fastDependencyScannerOutput.data(using: .utf8)!) driver.explicitModuleBuildHandler = try ExplicitModuleBuildHandler(dependencyGraph: moduleDependencyGraph, - toolchain: driver.toolchain) + toolchain: driver.toolchain, + fileSystem: localFileSystem) let modulePrebuildJobs = try driver.explicitModuleBuildHandler!.generateExplicitModuleDependenciesBuildJobs() XCTAssertEqual(modulePrebuildJobs.count, 4) @@ -258,4 +268,34 @@ final class ExplicitModuleBuildTests: XCTestCase { } #endif } + + func testExplicitSwiftModuleMap() throws { + let jsonExample : String = """ + [ + { + "moduleName": "A", + "modulePath": "A.swiftmodule", + "docPath": "A.swiftdoc", + "sourceInfoPath": "A.swiftsourceinfo" + }, + { + "moduleName": "B", + "modulePath": "B.swiftmodule", + "docPath": "B.swiftdoc", + "sourceInfoPath": "B.swiftsourceinfo" + } + ] + """ + let moduleMap = try JSONDecoder().decode(Array.self, + from: jsonExample.data(using: .utf8)!) + XCTAssertEqual(moduleMap.count, 2) + XCTAssertEqual(moduleMap[0].moduleName, "A") + XCTAssertEqual(moduleMap[0].modulePath, "A.swiftmodule") + XCTAssertEqual(moduleMap[0].docPath, "A.swiftdoc") + XCTAssertEqual(moduleMap[0].sourceInfoPath, "A.swiftsourceinfo") + XCTAssertEqual(moduleMap[1].moduleName, "B") + XCTAssertEqual(moduleMap[1].modulePath, "B.swiftmodule") + XCTAssertEqual(moduleMap[1].docPath, "B.swiftdoc") + XCTAssertEqual(moduleMap[1].sourceInfoPath, "B.swiftsourceinfo") + } }