Skip to content

Cleanup to prepare for emit-module-separately by default #790

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Aug 11, 2021
Merged

Cleanup to prepare for emit-module-separately by default #790

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2226855
[Tests] Make tests more resilient to a change in the jobs order
xymus Jun 15, 2021
6a96648
[Tests] Force the use of merge-module in merge-module tests
xymus Jun 15, 2021
538edad
[Tests] Tests expecting compile jobs shouldn't use -emit-module
xymus Jun 24, 2021
f6d723d
Feed the emit-module swiftmodule output to the linker
xymus Jun 15, 2021
4cc0b07
Skip dep and dia files with emit module when there's a full build sch…
xymus Jun 24, 2021
cc8954e
[Tests] Update testDOTFileEmission for emit-module-separately
xymus Jul 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion Sources/SwiftDriver/Jobs/EmitModuleJob.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,15 @@ extension Driver {
addSupplementalOutput(path: objcGeneratedHeaderPath, flag: "-emit-objc-header-path", type: .objcHeader)
addSupplementalOutput(path: tbdPath, flag: "-emit-tbd-path", type: .tbd)

if isMergeModule || shouldCreateEmitModuleJob {
if isMergeModule {
return
}

// Skip files created by other jobs when emitting a module and building at the same time
if shouldCreateEmitModuleJob && compilerOutputType != .swiftModule {
return
}

// Add outputs that can't be merged
addSupplementalOutput(path: serializedDiagnosticsFilePath, flag: "-serialize-diagnostics-path", type: .diagnostics)
if let dependenciesFilePath = dependenciesFilePath {
Expand Down
38 changes: 31 additions & 7 deletions Sources/SwiftDriver/Jobs/Planning.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,6 @@ extension Driver {

try addPrecompileModuleDependenciesJobs(addJob: addJobBeforeCompiles)
try addPrecompileBridgingHeaderJob(addJob: addJobBeforeCompiles)
try addEmitModuleJob(addJobBeforeCompiles: addJobBeforeCompiles,
addJobAfterCompiles: addJobAfterCompiles)
let linkerInputs = try addJobsFeedingLinker(
addJobBeforeCompiles: addJobBeforeCompiles,
addCompileJobGroup: addCompileJobGroup,
Expand Down Expand Up @@ -179,12 +177,13 @@ extension Driver {
)
}

private mutating func addEmitModuleJob(addJobBeforeCompiles: (Job) -> Void, addJobAfterCompiles: (Job) -> Void) throws {
private mutating func addEmitModuleJob(addJobBeforeCompiles: (Job) -> Void) throws -> Job? {
if shouldCreateEmitModuleJob {
let emitJob = try emitModuleJob()
addJobBeforeCompiles(emitJob)
try addVerifyJobs(emitModuleJob: emitJob, addJob: addJobAfterCompiles)
return emitJob
}
return nil
}

private mutating func addJobsFeedingLinker(
Expand Down Expand Up @@ -219,10 +218,32 @@ extension Driver {
}
}

// Ensure that only one job emits the module files and insert a verify swiftinterface job
var jobCreatingSwiftModule: Job? = nil
func addPostModuleFilesJobs(_ emitModuleJob: Job) throws {
// We should only emit module files from one job
assert(jobCreatingSwiftModule == nil)
jobCreatingSwiftModule = emitModuleJob

try addVerifyJobs(emitModuleJob: emitModuleJob, addJob: addJobAfterCompiles)
}

// Whole-module
if let compileJob = try addSingleCompileJobs(addJob: addJobBeforeCompiles,
addJobOutputs: addJobOutputs,
emitModuleTrace: loadedModuleTracePath != nil) {
try addVerifyJobs(emitModuleJob: compileJob, addJob: addJobAfterCompiles)
try addPostModuleFilesJobs(compileJob)
}

// Emit-module-separately
if let emitModuleJob = try addEmitModuleJob(addJobBeforeCompiles: addJobBeforeCompiles) {
try addPostModuleFilesJobs(emitModuleJob)

try addWrapJobOrMergeOutputs(
mergeJob: emitModuleJob,
debugInfo: debugInfo,
addJob: addJobAfterCompiles,
addLinkerInput: addLinkerInput)
}

try addJobsForPrimaryInputs(
Expand All @@ -235,11 +256,13 @@ extension Driver {
addLinkerInput: addLinkerInput,
addJob: addJobAfterCompiles)

// Merge-module
if let mergeJob = try mergeModuleJob(
moduleInputs: moduleInputs,
moduleInputsFromJobOutputs: moduleInputsFromJobOutputs) {
addJobAfterCompiles(mergeJob)
try addVerifyJobs(emitModuleJob: mergeJob, addJob: addJobAfterCompiles)
try addPostModuleFilesJobs(mergeJob)

try addWrapJobOrMergeOutputs(
mergeJob: mergeJob,
debugInfo: debugInfo,
Expand Down Expand Up @@ -394,7 +417,8 @@ extension Driver {
) throws -> Job? {
guard moduleOutputInfo.output != nil,
!(moduleInputs.isEmpty && moduleInputsFromJobOutputs.isEmpty),
compilerMode.usesPrimaryFileInputs
compilerMode.usesPrimaryFileInputs,
!shouldCreateEmitModuleJob
else { return nil }
return try mergeModuleJob(inputs: moduleInputs, inputsFromOutputs: moduleInputsFromJobOutputs)
}
Expand Down
6 changes: 3 additions & 3 deletions Tests/SwiftDriverTests/ParsableMessageTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ final class ParsableMessageTests: XCTestCase {
try withTemporaryDirectory { path in
try withHijackedBufferedErrorStream(in: path) { errorBuffer in
let resolver = try ArgsResolver(fileSystem: localFileSystem)
var driver = try Driver(args: ["swiftc", "-emit-module", "-o", "test.swiftmodule",
var driver = try Driver(args: ["swiftc", "-o", "test.o",
"main.swift", "test1.swift", "test2.swift",
"-enable-batch-mode", "-driver-batch-count", "1",
"-working-directory", "/WorkDir"])
Expand Down Expand Up @@ -201,7 +201,7 @@ final class ParsableMessageTests: XCTestCase {
var toolDelegate: ToolExecutionDelegate?
try withHijackedBufferedErrorStream(in: path) { errorBuffer in
let resolver = try ArgsResolver(fileSystem: localFileSystem)
var driver = try Driver(args: ["swiftc", "-emit-module", "-o", "test.swiftmodule",
var driver = try Driver(args: ["swiftc", "-o", "test.o",
"main.swift", "test1.swift", "test2.swift",
"-enable-batch-mode", "-driver-batch-count", "1",
"-working-directory", "/WorkDir"])
Expand Down Expand Up @@ -278,7 +278,7 @@ final class ParsableMessageTests: XCTestCase {
var toolDelegate: ToolExecutionDelegate?
try withHijackedBufferedErrorStream(in: path) { errorBuffer in
let resolver = try ArgsResolver(fileSystem: localFileSystem)
var driver = try Driver(args: ["swiftc", "-emit-module", "-o", "test.swiftmodule",
var driver = try Driver(args: ["swiftc", "-o", "test.o",
"main.swift", "test1.swift", "test2.swift",
"-enable-batch-mode", "-driver-batch-count", "1",
"-working-directory", "/WorkDir"])
Expand Down
108 changes: 55 additions & 53 deletions Tests/SwiftDriverTests/SwiftDriverTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -514,14 +514,15 @@ final class SwiftDriverTests: XCTestCase {
// Forwarding of arguments.
var driver2 = try Driver(args: ["swiftc", "-color-diagnostics", "foo.swift", "bar.swift", "-working-directory", "/tmp", "-api-diff-data-file", "diff.txt", "-Xfrontend", "-HI", "-no-color-diagnostics", "-g"])
let plannedJobs2 = try driver2.planBuild()
XCTAssert(plannedJobs2[0].commandLine.contains(Job.ArgTemplate.path(.absolute(try AbsolutePath(validating: "/tmp/diff.txt")))))
XCTAssert(plannedJobs2[0].commandLine.contains(.flag("-HI")))
XCTAssert(!plannedJobs2[0].commandLine.contains(.flag("-Xfrontend")))
XCTAssert(plannedJobs2[0].commandLine.contains(.flag("-no-color-diagnostics")))
XCTAssert(!plannedJobs2[0].commandLine.contains(.flag("-color-diagnostics")))
XCTAssert(plannedJobs2[0].commandLine.contains(.flag("-target")))
XCTAssert(plannedJobs2[0].commandLine.contains(.flag(driver2.targetTriple.triple)))
XCTAssert(plannedJobs2[0].commandLine.contains(.flag("-enable-anonymous-context-mangled-names")))
let compileJob = plannedJobs2.first(where: {$0.kind == .compile})!
XCTAssert(compileJob.commandLine.contains(Job.ArgTemplate.path(.absolute(try AbsolutePath(validating: "/tmp/diff.txt")))))
XCTAssert(compileJob.commandLine.contains(.flag("-HI")))
XCTAssert(!compileJob.commandLine.contains(.flag("-Xfrontend")))
XCTAssert(compileJob.commandLine.contains(.flag("-no-color-diagnostics")))
XCTAssert(!compileJob.commandLine.contains(.flag("-color-diagnostics")))
XCTAssert(compileJob.commandLine.contains(.flag("-target")))
XCTAssert(compileJob.commandLine.contains(.flag(driver2.targetTriple.triple)))
XCTAssert(compileJob.commandLine.contains(.flag("-enable-anonymous-context-mangled-names")))

var driver3 = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-emit-library", "-module-name", "Test"])
let plannedJobs3 = try driver3.planBuild()
Expand Down Expand Up @@ -757,7 +758,7 @@ final class SwiftDriverTests: XCTestCase {
}

func testMergeModuleEmittingDependencies() throws {
var driver1 = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Foo", "-emit-dependencies", "-emit-module", "-serialize-diagnostics", "-driver-filelist-threshold=9999"])
var driver1 = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Foo", "-emit-dependencies", "-emit-module", "-serialize-diagnostics", "-driver-filelist-threshold=9999", "-no-emit-module-separately"])
let plannedJobs = try driver1.planBuild().removingAutolinkExtractJobs()
XCTAssertTrue(plannedJobs[0].kind == .compile)
XCTAssertTrue(plannedJobs[1].kind == .compile)
Expand All @@ -770,6 +771,15 @@ final class SwiftDriverTests: XCTestCase {
XCTAssertFalse(plannedJobs[2].commandLine.contains(.flag("-serialize-diagnostics-path")))
}

func testEmitModuleEmittingDependencies() throws {
var driver1 = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Foo", "-emit-dependencies", "-emit-module", "-serialize-diagnostics", "-driver-filelist-threshold=9999", "-experimental-emit-module-separately"])
let plannedJobs = try driver1.planBuild().removingAutolinkExtractJobs()
XCTAssertEqual(plannedJobs.count, 1)
XCTAssertTrue(plannedJobs[0].kind == .emitModule)
XCTAssertTrue(plannedJobs[0].commandLine.contains(.flag("-emit-dependencies-path")))
XCTAssertTrue(plannedJobs[0].commandLine.contains(.flag("-serialize-diagnostics-path")))
}

func testReferenceDependencies() throws {
var driver = try Driver(args: ["swiftc", "foo.swift", "-incremental"])
let plannedJobs = try driver.planBuild()
Expand Down Expand Up @@ -2112,7 +2122,7 @@ final class SwiftDriverTests: XCTestCase {

func testMergeModulesOnly() throws {
do {
var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Test", "-emit-module", "-disable-bridging-pch", "-import-objc-header", "TestInputHeader.h", "-emit-dependencies", "-emit-module-source-info-path", "/foo/bar/Test.swiftsourceinfo"])
var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Test", "-emit-module", "-disable-bridging-pch", "-import-objc-header", "TestInputHeader.h", "-emit-dependencies", "-emit-module-source-info-path", "/foo/bar/Test.swiftsourceinfo", "-no-emit-module-separately"])
let plannedJobs = try driver.planBuild()
XCTAssertEqual(plannedJobs.count, 3)
XCTAssertEqual(Set(plannedJobs.map { $0.kind }), Set([.compile, .mergeModule]))
Expand Down Expand Up @@ -2140,7 +2150,7 @@ final class SwiftDriverTests: XCTestCase {
}

do {
var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Test", "-emit-module-path", "/foo/bar/Test.swiftmodule" ])
var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Test", "-emit-module-path", "/foo/bar/Test.swiftmodule", "-no-emit-module-separately"])
let plannedJobs = try driver.planBuild()
XCTAssertEqual(plannedJobs.count, 3)
XCTAssertTrue(plannedJobs[2].tool.name.contains("swift"))
Expand All @@ -2152,7 +2162,7 @@ final class SwiftDriverTests: XCTestCase {

do {
// Make sure the swiftdoc path is correct for a relative module
var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Test", "-emit-module-path", "Test.swiftmodule" ])
var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Test", "-emit-module-path", "Test.swiftmodule", "-no-emit-module-separately"])
let plannedJobs = try driver.planBuild()
XCTAssertEqual(plannedJobs.count, 3)
XCTAssertTrue(plannedJobs[2].tool.name.contains("swift"))
Expand All @@ -2164,7 +2174,7 @@ final class SwiftDriverTests: XCTestCase {

do {
// Make sure the swiftdoc path is correct for an inferred module
var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Test", "-emit-module"])
var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Test", "-emit-module", "-no-emit-module-separately"])
let plannedJobs = try driver.planBuild()
XCTAssertEqual(plannedJobs.count, 3)
XCTAssertTrue(plannedJobs[2].tool.name.contains("swift"))
Expand All @@ -2176,7 +2186,7 @@ final class SwiftDriverTests: XCTestCase {

do {
// -o specified
var driver = try Driver(args: ["swiftc", "-emit-module", "-o", "/tmp/test.swiftmodule", "input.swift"])
var driver = try Driver(args: ["swiftc", "-emit-module", "-o", "/tmp/test.swiftmodule", "input.swift", "-no-emit-module-separately"])
let plannedJobs = try driver.planBuild()

XCTAssertEqual(plannedJobs.count, 2)
Expand Down Expand Up @@ -3370,7 +3380,7 @@ final class SwiftDriverTests: XCTestCase {
// Reset the temporary store to ensure predictable results.
VirtualPath.resetTemporaryFileStore()
var driver = try Driver(args: [
"swiftc", "-emit-executable", "test.swift", "-emit-module", "-avoid-emit-module-source-info"
"swiftc", "-emit-executable", "test.swift", "-emit-module", "-avoid-emit-module-source-info", "-experimental-emit-module-separately"
])
let plannedJobs = try driver.planBuild()

Expand All @@ -3383,25 +3393,21 @@ final class SwiftDriverTests: XCTestCase {
XCTAssertEqual(output,
"""
digraph Jobs {
"compile (swift-frontend)" [style=bold];
"emitModule (swift-frontend)" [style=bold];
"test.swift" [fontsize=12];
"test.swift" -> "emitModule (swift-frontend)" [color=blue];
"test.swiftmodule" [fontsize=12];
"emitModule (swift-frontend)" -> "test.swiftmodule" [color=green];
"test.swiftdoc" [fontsize=12];
"emitModule (swift-frontend)" -> "test.swiftdoc" [color=green];
"compile (swift-frontend)" [style=bold];
"test.swift" -> "compile (swift-frontend)" [color=blue];
"test-1.o" [fontsize=12];
"compile (swift-frontend)" -> "test-1.o" [color=green];
"test-1.swiftmodule" [fontsize=12];
"compile (swift-frontend)" -> "test-1.swiftmodule" [color=green];
"test-1.swiftdoc" [fontsize=12];
"compile (swift-frontend)" -> "test-1.swiftdoc" [color=green];
"autolinkExtract (swift-autolink-extract)" [style=bold];
"test-1.o" -> "autolinkExtract (swift-autolink-extract)" [color=blue];
"test-2.autolink" [fontsize=12];
"autolinkExtract (swift-autolink-extract)" -> "test-2.autolink" [color=green];
"mergeModule (swift-frontend)" [style=bold];
"test-1.swiftmodule" -> "mergeModule (swift-frontend)" [color=blue];
"test.swiftmodule" [fontsize=12];
"mergeModule (swift-frontend)" -> "test.swiftmodule" [color=green];
"test.swiftdoc" [fontsize=12];
"mergeModule (swift-frontend)" -> "test.swiftdoc" [color=green];
"link (clang)" [style=bold];
"test-1.o" -> "link (clang)" [color=blue];
"test-2.autolink" -> "link (clang)" [color=blue];
Expand All @@ -3414,21 +3420,17 @@ final class SwiftDriverTests: XCTestCase {
XCTAssertEqual(output,
"""
digraph Jobs {
"compile (swift-frontend)" [style=bold];
"emitModule (swift-frontend)" [style=bold];
"test.swift" [fontsize=12];
"test.swift" -> "emitModule (swift-frontend)" [color=blue];
"test.swiftmodule" [fontsize=12];
"emitModule (swift-frontend)" -> "test.swiftmodule" [color=green];
"test.swiftdoc" [fontsize=12];
"emitModule (swift-frontend)" -> "test.swiftdoc" [color=green];
"compile (swift-frontend)" [style=bold];
"test.swift" -> "compile (swift-frontend)" [color=blue];
"test-1.o" [fontsize=12];
"compile (swift-frontend)" -> "test-1.o" [color=green];
"test-1.swiftmodule" [fontsize=12];
"compile (swift-frontend)" -> "test-1.swiftmodule" [color=green];
"test-1.swiftdoc" [fontsize=12];
"compile (swift-frontend)" -> "test-1.swiftdoc" [color=green];
"mergeModule (swift-frontend)" [style=bold];
"test-1.swiftmodule" -> "mergeModule (swift-frontend)" [color=blue];
"test.swiftmodule" [fontsize=12];
"mergeModule (swift-frontend)" -> "test.swiftmodule" [color=green];
"test.swiftdoc" [fontsize=12];
"mergeModule (swift-frontend)" -> "test.swiftdoc" [color=green];
"link (\(dynamicLinker))" [style=bold];
"test-1.o" -> "link (\(dynamicLinker))" [color=blue];
"test" [fontsize=12];
Expand Down Expand Up @@ -4269,7 +4271,7 @@ final class SwiftDriverTests: XCTestCase {

// Ensure the merge-module step is not passed the precompiled header
do {
var driver = try Driver(args: ["swiftc", "-emit-module", "-import-objc-header", "header.h", "foo.swift"])
var driver = try Driver(args: ["swiftc", "-emit-module", "-import-objc-header", "header.h", "foo.swift", "-no-emit-module-separately"])
let plannedJobs = try driver.planBuild()
XCTAssertEqual(plannedJobs.count, 3)

Expand Down Expand Up @@ -4409,20 +4411,20 @@ final class SwiftDriverTests: XCTestCase {
let plannedJobs = try driver.planBuild().removingAutolinkExtractJobs()
XCTAssertEqual(plannedJobs.count, 3)

XCTAssertEqual(plannedJobs[0].kind, .compile)
XCTAssertEqual(plannedJobs[0].inputs.count, 1)
XCTAssertEqual(plannedJobs[0].inputs[0].file, .relative(RelativePath("embed-bitcode.swift")))
XCTAssertEqual(plannedJobs[0].outputs.count, 4)
XCTAssertTrue(matchTemporary(plannedJobs[0].outputs[0].file, "embed-bitcode.bc"))
XCTAssertTrue(matchTemporary(plannedJobs[0].outputs[1].file, "embed-bitcode.swiftmodule"))
XCTAssertTrue(matchTemporary(plannedJobs[0].outputs[2].file, "embed-bitcode.swiftdoc"))
XCTAssertTrue(matchTemporary(plannedJobs[0].outputs[3].file, "embed-bitcode.swiftsourceinfo"))

XCTAssertEqual(plannedJobs[1].kind, .backend)
XCTAssertEqual(plannedJobs[1].inputs.count, 1)
XCTAssertTrue(matchTemporary(plannedJobs[1].inputs[0].file, "embed-bitcode.bc"))
XCTAssertEqual(plannedJobs[1].outputs.count, 1)
XCTAssertEqual(plannedJobs[1].outputs[0].file, .relative(RelativePath("embed-bitcode.o")))
let compileJob = plannedJobs.first(where: {$0.kind == .compile})!
XCTAssertEqual(compileJob.inputs.count, 1)
XCTAssertEqual(compileJob.inputs[0].file, .relative(RelativePath("embed-bitcode.swift")))
XCTAssertEqual(compileJob.outputs.count, 4)
XCTAssertTrue(matchTemporary(compileJob.outputs[0].file, "embed-bitcode.bc"))
XCTAssertTrue(matchTemporary(compileJob.outputs[1].file, "embed-bitcode.swiftmodule"))
XCTAssertTrue(matchTemporary(compileJob.outputs[2].file, "embed-bitcode.swiftdoc"))
XCTAssertTrue(matchTemporary(compileJob.outputs[3].file, "embed-bitcode.swiftsourceinfo"))

let backendJob = plannedJobs.first(where: {$0.kind == .backend})!
XCTAssertEqual(backendJob.inputs.count, 1)
XCTAssertTrue(matchTemporary(backendJob.inputs[0].file, "embed-bitcode.bc"))
XCTAssertEqual(backendJob.outputs.count, 1)
XCTAssertEqual(backendJob.outputs[0].file, .relative(RelativePath("embed-bitcode.o")))

XCTAssertEqual(plannedJobs[2].kind, .mergeModule)
XCTAssertEqual(plannedJobs[2].inputs.count, 1)
Expand Down Expand Up @@ -4871,7 +4873,7 @@ final class SwiftDriverTests: XCTestCase {

func testFilelist() throws {
do {
var driver = try Driver(args: ["swiftc", "-emit-module", "./a.swift", "./b.swift", "./c.swift", "-module-name", "main", "-target", "x86_64-apple-macosx10.9", "-driver-filelist-threshold=0"])
var driver = try Driver(args: ["swiftc", "-emit-module", "./a.swift", "./b.swift", "./c.swift", "-module-name", "main", "-target", "x86_64-apple-macosx10.9", "-driver-filelist-threshold=0", "-no-emit-module-separately"])
let plannedJobs = try driver.planBuild()

let jobA = plannedJobs[0]
Expand Down