SwiftDriver performance fixes for swift caching

Sources/SwiftDriver/Jobs/CompileJob.swift

@@ -231,7 +231,8 @@ extension Driver {
                            outputType: FileType?,
                            addJobOutputs: ([TypedVirtualPath]) -> Void,
                            pchCompileJob: Job?,
-                           emitModuleTrace: Bool)
+                           emitModuleTrace: Bool,
+                           produceCacheKey: Bool)
   throws -> Job {
     var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
     var inputs: [TypedVirtualPath] = []

Sources/SwiftDriver/Jobs/Planning.swift

@@ -90,16 +90,17 @@ extension Driver {
       incrementalCompilationState = nil
     }
 
-    return try (
-      // For compatibility with swiftpm, the driver produces batched jobs
-      // for every job, even when run in incremental mode, so that all jobs
-      // can be returned from `planBuild`.
-      // But in that case, don't emit lifecycle messages.
-      formBatchedJobs(jobsInPhases.allJobs,
-                      showJobLifecycle: showJobLifecycle && incrementalCompilationState == nil,
-                      jobCreatingPch: jobsInPhases.allJobs.first(where: {$0.kind == .generatePCH})),
-      incrementalCompilationState
-    )
+    let batchedJobs: [Job]
+    // If the jobs are batched during the incremental build, reuse the computation rather than computing the batches again.
+    if let incrementalState = incrementalCompilationState {
+      batchedJobs = incrementalState.mandatoryJobsInOrder + incrementalState.jobsAfterCompiles
+    } else {
+      batchedJobs = try formBatchedJobs(jobsInPhases.allJobs,
+                                        showJobLifecycle: showJobLifecycle,
+                                        jobCreatingPch: jobsInPhases.allJobs.first(where: {$0.kind == .generatePCH}))
+    }
+
+    return (batchedJobs, incrementalCompilationState)
   }
 
   /// If performing an explicit module build, compute an inter-module dependency graph.
@@ -355,7 +356,8 @@ extension Driver {
                                  outputType: compilerOutputType,
                                  addJobOutputs: addJobOutputs,
                                  pchCompileJob: pchCompileJob,
-                                 emitModuleTrace: emitModuleTrace)
+                                 emitModuleTrace: emitModuleTrace,
+                                 produceCacheKey: true)
     addJob(compile)
     return compile
   }
@@ -446,11 +448,14 @@ extension Driver {
     // We can skip the compile jobs if all we want is a module when it's
     // built separately.
     if parsedOptions.hasArgument(.driverExplicitModuleBuild), canSkipIfOnlyModule { return }
+    // If we are in the batch mode, the constructed jobs here will be batched
+    // later. There is no need to produce cache key for the job.
     let compile = try compileJob(primaryInputs: [primaryInput],
                                  outputType: compilerOutputType,
                                  addJobOutputs: addJobOutputs,
                                  pchCompileJob: pchCompileJob,
-                                 emitModuleTrace: emitModuleTrace)
+                                 emitModuleTrace: emitModuleTrace,
+                                 produceCacheKey: !compilerMode.isBatchCompile)
     addCompileJob(compile)
   }
 
@@ -872,7 +877,8 @@ extension Driver {
                             outputType: compilerOutputType,
                             addJobOutputs: {_ in },
                             pchCompileJob: jobCreatingPch,
-                            emitModuleTrace: constituentsEmittedModuleTrace)
+                            emitModuleTrace: constituentsEmittedModuleTrace,
+                            produceCacheKey: true)
     }
     return batchedCompileJobs + noncompileJobs
   }

Sources/SwiftDriver/SwiftScan/SwiftScan.swift

@@ -674,20 +674,45 @@ private extension swiftscan_functions_t {
   }
 }
 
-// TODO: Move to TSC?
-/// Perform an  `action` passing it a `const char **` constructed out of `[String]`
-@_spi(Testing) public func withArrayOfCStrings<T>(_ strings: [String],
-                                                  _ action:  (UnsafeMutablePointer<UnsafePointer<Int8>?>?) -> T) -> T
-{
-#if os(Windows)
-  let cstrings = strings.map { _strdup($0) } + [nil]
-#else
-  let cstrings = strings.map { strdup($0) } + [nil]
-#endif
-  let unsafeCStrings = cstrings.map { UnsafePointer($0) }
-  let result = unsafeCStrings.withUnsafeBufferPointer {
-    action(UnsafeMutablePointer(mutating: $0.baseAddress))
+// TODO: Move the following functions to TSC?
+/// Helper function to scan a sequence type to help generate pointers for C String Arrays.
+func scan<
+  S: Sequence, U
+>(_ seq: S, _ initial: U, _ combine: (U, S.Element) -> U) -> [U] {
+  var result: [U] = []
+  result.reserveCapacity(seq.underestimatedCount)
+  var runningResult = initial
+  for element in seq {
+    runningResult = combine(runningResult, element)
+    result.append(runningResult)
   }
-  for ptr in cstrings { if let ptr = ptr { free(ptr) } }
   return result
 }
+
+/// Perform an  `action` passing it a `const char **` constructed out of `[String]`
+@_spi(Testing) public func withArrayOfCStrings<T>(
+  _ args: [String],
+  _ body: (UnsafeMutablePointer<UnsafePointer<Int8>?>?) -> T
+) -> T {
+  let argsCounts = Array(args.map { $0.utf8.count + 1 })
+  let argsOffsets = [0] + scan(argsCounts, 0, +)
+  let argsBufferSize = argsOffsets.last!
+  var argsBuffer: [UInt8] = []
+  argsBuffer.reserveCapacity(argsBufferSize)
+  for arg in args {
+    argsBuffer.append(contentsOf: arg.utf8)
+    argsBuffer.append(0)
+  }
+  return argsBuffer.withUnsafeMutableBufferPointer {
+    (argsBuffer) in
+    let ptr = UnsafeRawPointer(argsBuffer.baseAddress!).bindMemory(
+      to: Int8.self, capacity: argsBuffer.count)
+    var cStrings: [UnsafePointer<Int8>?] = argsOffsets.map { ptr + $0 }
+    cStrings[cStrings.count - 1] = nil
+    return cStrings.withUnsafeMutableBufferPointer {
+      let unsafeString = UnsafeMutableRawPointer($0.baseAddress!).bindMemory(
+        to: UnsafePointer<Int8>?.self, capacity: $0.count)
+      return body(unsafeString)
+    }
+  }
+}

Tests/SwiftDriverTests/CachingBuildTests.swift

@@ -986,17 +986,60 @@ final class CachingBuildTests: XCTestCase {
         XCTFail("Cached compilation doesn't have a CAS")
       }
       try checkCASForResults(jobs: jobs, cas: cas, fs: driver.fileSystem)
+    }
+  }
 
-      // try replan the job and make sure some key command-line options are generated.
-      let rebuildJobs = try driver.planBuild()
-      for job in rebuildJobs {
-        if job.kind == .compile || job.kind == .emitModule {
-          XCTAssertTrue(job.commandLine.contains(.flag(String("-disable-implicit-swift-modules"))))
-          XCTAssertTrue(job.commandLine.contains(.flag(String("-cache-compile-job"))))
-          XCTAssertTrue(job.commandLine.contains(.flag(String("-cas-path"))))
-          XCTAssertTrue(job.commandLine.contains(.flag(String("-bridging-header-pch-key"))))
-        }
+  func testCacheBatchBuildPlan() throws {
+    try withTemporaryDirectory { path in
+      try localFileSystem.changeCurrentWorkingDirectory(to: path)
+      let moduleCachePath = path.appending(component: "ModuleCache")
+      let casPath = path.appending(component: "cas")
+      try localFileSystem.createDirectory(moduleCachePath)
+      let main = path.appending(component: "testCachingBuild.swift")
+      let mainFileContent = "import C;import E;import G;"
+      try localFileSystem.writeFileContents(main) {
+        $0.send(mainFileContent)
       }
+      let ofm = path.appending(component: "ofm.json")
+      let inputPathsAndContents: [(AbsolutePath, String)] = [(main, mainFileContent)]
+      OutputFileMapCreator.write(
+        module: "Test", inputPaths: inputPathsAndContents.map {$0.0},
+        derivedData: path, to: ofm, excludeMainEntry: false)
+
+      let cHeadersPath: AbsolutePath =
+          try testInputsPath.appending(component: "ExplicitModuleBuilds")
+                            .appending(component: "CHeaders")
+      let swiftModuleInterfacesPath: AbsolutePath =
+          try testInputsPath.appending(component: "ExplicitModuleBuilds")
+                            .appending(component: "Swift")
+      let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? []
+      let bridgingHeaderpath: AbsolutePath =
+          cHeadersPath.appending(component: "Bridging.h")
+      var driver = try Driver(args: ["swiftc",
+                                     "-I", cHeadersPath.nativePathString(escaped: true),
+                                     "-I", swiftModuleInterfacesPath.nativePathString(escaped: true),
+                                     "-explicit-module-build", "-Rcache-compile-job", "-incremental",
+                                     "-module-cache-path", moduleCachePath.nativePathString(escaped: true),
+                                     "-cache-compile-job", "-cas-path", casPath.nativePathString(escaped: true),
+                                     "-import-objc-header", bridgingHeaderpath.nativePathString(escaped: true),
+                                     "-output-file-map", ofm.nativePathString(escaped: true),
+                                     "-working-directory", path.nativePathString(escaped: true),
+                                     main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
+                              interModuleDependencyOracle: dependencyOracle)
+      let jobs = try driver.planBuild()
+      try driver.run(jobs: jobs)
+      XCTAssertFalse(driver.diagnosticEngine.hasErrors)
+
+      let scanLibPath = try XCTUnwrap(driver.getSwiftScanLibPath())
+      try dependencyOracle.verifyOrCreateScannerInstance(swiftScanLibPath: scanLibPath)
+
+      let cas = try dependencyOracle.getOrCreateCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: [])
+      if let driverCAS = driver.cas {
+        XCTAssertEqual(cas, driverCAS, "CAS should only be created once")
+      } else {
+        XCTFail("Cached compilation doesn't have a CAS")
+      }
+      try checkCASForResults(jobs: jobs, cas: cas, fs: driver.fileSystem)
     }
   }