Merge pull request #1498 from cachemeifyoucan/eng/PR-swift-caching-incremental-build-bridging-header

[Caching] Fix incremental cache build with bridging header

Author: cachemeifyoucan

Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift

@@ -83,6 +83,7 @@ extension IncrementalCompilationState.FirstWaveComputer {
       Dictionary(uniqueKeysWithValues:
                   jobsInPhases.compileGroups.map { ($0.primaryInput, $0) })
     let buildRecord = self.moduleDependencyGraph.buildRecord
+    let jobCreatingPch = jobsInPhases.beforeCompiles.first(where: {$0.kind == .generatePCH})
     guard !buildRecord.inputInfos.isEmpty else {
       func everythingIsMandatory()
         throws -> (initiallySkippedCompileGroups: [TypedVirtualPath: CompileJobGroup],
@@ -97,7 +98,8 @@ extension IncrementalCompilationState.FirstWaveComputer {
         jobsInPhases.beforeCompiles +
         batchJobFormer.formBatchedJobs(
           mandatoryCompileGroupsInOrder.flatMap {$0.allJobs()},
-          showJobLifecycle: showJobLifecycle)
+          showJobLifecycle: showJobLifecycle,
+          jobCreatingPch: jobCreatingPch)
 
         moduleDependencyGraph.setPhase(to: .buildingAfterEachCompilation)
         return (initiallySkippedCompileGroups: [:],
@@ -124,7 +126,8 @@ extension IncrementalCompilationState.FirstWaveComputer {
     let mandatoryBeforeCompilesJobs = try computeMandatoryBeforeCompilesJobs()
     let batchedCompilationJobs = try batchJobFormer.formBatchedJobs(
       mandatoryCompileGroupsInOrder.flatMap {$0.allJobs()},
-      showJobLifecycle: showJobLifecycle)
+      showJobLifecycle: showJobLifecycle,
+      jobCreatingPch: jobCreatingPch)
 
     // In the case where there are no compilation jobs to run on this build (no source-files were changed),
     // we can skip running `beforeCompiles` jobs if we also ensure that none of the `afterCompiles` jobs

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationProtectedState.swift

@@ -34,14 +34,17 @@ extension IncrementalCompilationState {
     /// fileprivate in order to control concurrency.
     fileprivate let moduleDependencyGraph: ModuleDependencyGraph
 
+    fileprivate let jobCreatingPch: Job?
     fileprivate let reporter: Reporter?
 
     init(skippedCompileGroups: [TypedVirtualPath: CompileJobGroup],
          _ moduleDependencyGraph: ModuleDependencyGraph,
+         _ jobCreatingPch: Job?,
          _ driver: inout Driver) {
       self.skippedCompileGroups = skippedCompileGroups
       self.moduleDependencyGraph = moduleDependencyGraph
       self.reporter = moduleDependencyGraph.info.reporter
+      self.jobCreatingPch = jobCreatingPch
       self.driver = driver
     }
   }
@@ -61,7 +64,7 @@ extension IncrementalCompilationState.ProtectedState {
     mutationSafetyPrecondition()
     // batch in here to protect the Driver from concurrent access
     return try collectUnbatchedJobsDiscoveredToBeNeededAfterFinishing(job: finishedJob)
-      .map {try driver.formBatchedJobs($0, showJobLifecycle: driver.showJobLifecycle)}
+      .map {try driver.formBatchedJobs($0, showJobLifecycle: driver.showJobLifecycle, jobCreatingPch: jobCreatingPch)}
   }
 
   /// Remember a job (group) that is before a compile or a compile itself.

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift

@@ -77,6 +77,7 @@ public final class IncrementalCompilationState {
     self.protectedState = ProtectedState(
       skippedCompileGroups: firstWave.initiallySkippedCompileGroups,
       initialState.graph,
+      jobsInPhases.allJobs.first(where: {$0.kind == .generatePCH}),
       &driver)
     self.mandatoryJobsInOrder = firstWave.mandatoryJobsInOrder
     self.jobsAfterCompiles = jobsInPhases.afterCompiles

Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift

@@ -453,14 +453,12 @@ extension Driver {
                                                       pchCompileJob: Job?) throws {
     guard let pchJob = pchCompileJob, isCachingEnabled else { return }
 
-    // The pch input file (the bridging header) is added as last inputs to the job.
-    guard let inputFile = pchJob.inputs.last else { assertionFailure("no input files from pch job"); return }
-    assert(inputFile.type == .objcHeader, "Expect objc header input type")
-    let bridgingHeaderCacheKey = try computeOutputCacheKey(commandLine: pchJob.commandLine,
-                                                           input: inputFile)
-    guard let key = bridgingHeaderCacheKey else { return }
+    assert(pchJob.outputCacheKeys.count == 1, "Expect one and only one cache key from pch job")
+    guard let bridgingHeaderCacheKey = pchJob.outputCacheKeys.first?.value else {
+      throw Error.unsupportedConfigurationForCaching("pch job doesn't have an associated cache key")
+    }
     commandLine.appendFlag("-bridging-header-pch-key")
-    commandLine.appendFlag(key)
+    commandLine.appendFlag(bridgingHeaderCacheKey)
   }
 
   mutating func addFrontendSupplementaryOutputArguments(commandLine: inout [Job.ArgTemplate],

Sources/SwiftDriver/Jobs/Planning.swift

@@ -127,7 +127,8 @@ extension Driver {
       // can be returned from `planBuild`.
       // But in that case, don't emit lifecycle messages.
       formBatchedJobs(jobsInPhases.allJobs,
-                      showJobLifecycle: showJobLifecycle && incrementalCompilationState == nil),
+                      showJobLifecycle: showJobLifecycle && incrementalCompilationState == nil,
+                      jobCreatingPch: jobsInPhases.allJobs.first(where: {$0.kind == .generatePCH})),
       incrementalCompilationState
     )
   }
@@ -838,7 +839,7 @@ extension Driver {
   ///
   /// So, in order to avoid making jobs and rebatching, the code would have to just get outputs for each
   /// compilation. But `compileJob` intermixes the output computation with other stuff.
-  mutating func formBatchedJobs(_ jobs: [Job], showJobLifecycle: Bool) throws -> [Job] {
+  mutating func formBatchedJobs(_ jobs: [Job], showJobLifecycle: Bool, jobCreatingPch: Job?) throws -> [Job] {
     guard compilerMode.isBatchCompile else {
       // Don't even go through the logic so as to not print out confusing
       // "batched foobar" messages.
@@ -864,7 +865,6 @@ extension Driver {
       compileJobs.filter { $0.outputs.contains {$0.type == .moduleTrace} }
         .flatMap {$0.primaryInputs}
     )
-    let jobCreatingPch: Job? = jobs.first(where: {$0.kind == .generatePCH})
 
     let batchedCompileJobs = try inputsInOrder.compactMap { anInput -> Job? in
       let idx = partitions.assignment[anInput]!

Tests/SwiftDriverTests/CachingBuildTests.swift

@@ -761,4 +761,73 @@ final class CachingBuildTests: XCTestCase {
       }
     }
   }
+
+  func testCacheIncrementalBuildPlan() throws {
+    try withTemporaryDirectory { path in
+      try localFileSystem.changeCurrentWorkingDirectory(to: path)
+      let moduleCachePath = path.appending(component: "ModuleCache")
+      let casPath = path.appending(component: "cas")
+      try localFileSystem.createDirectory(moduleCachePath)
+      let main = path.appending(component: "testCachingBuild.swift")
+      let mainFileContent = "import C;import E;import G;"
+      try localFileSystem.writeFileContents(main) {
+        $0.send(mainFileContent)
+      }
+      let ofm = path.appending(component: "ofm.json")
+      let inputPathsAndContents: [(AbsolutePath, String)] = [(main, mainFileContent)]
+      OutputFileMapCreator.write(
+        module: "Test", inputPaths: inputPathsAndContents.map {$0.0},
+        derivedData: path, to: ofm, excludeMainEntry: false)
+
+      let cHeadersPath: AbsolutePath =
+          try testInputsPath.appending(component: "ExplicitModuleBuilds")
+                            .appending(component: "CHeaders")
+      let swiftModuleInterfacesPath: AbsolutePath =
+          try testInputsPath.appending(component: "ExplicitModuleBuilds")
+                            .appending(component: "Swift")
+      let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? []
+      let bridgingHeaderpath: AbsolutePath =
+          cHeadersPath.appending(component: "Bridging.h")
+      var driver = try Driver(args: ["swiftc",
+                                     "-I", cHeadersPath.nativePathString(escaped: true),
+                                     "-I", swiftModuleInterfacesPath.nativePathString(escaped: true),
+                                     "-explicit-module-build", "-v", "-Rcache-compile-job", "-incremental",
+                                     "-module-cache-path", moduleCachePath.nativePathString(escaped: true),
+                                     "-cache-compile-job", "-cas-path", casPath.nativePathString(escaped: true),
+                                     "-import-objc-header", bridgingHeaderpath.nativePathString(escaped: true),
+                                     "-output-file-map", ofm.nativePathString(escaped: true),
+                                     "-working-directory", path.nativePathString(escaped: true),
+                                     main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
+                              env: ProcessEnv.vars)
+      guard driver.isFeatureSupported(.cache_compile_job) else {
+        throw XCTSkip("toolchain does not support caching.")
+      }
+      let jobs = try driver.planBuild()
+      try driver.run(jobs: jobs)
+      XCTAssertFalse(driver.diagnosticEngine.hasErrors)
+
+      let dependencyOracle = InterModuleDependencyOracle()
+      let scanLibPath = try XCTUnwrap(driver.toolchain.lookupSwiftScanLib())
+      guard try dependencyOracle
+              .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
+                                             swiftScanLibPath: scanLibPath) else {
+        XCTFail("Dependency scanner library not found")
+        return
+      }
+
+      let cas = try dependencyOracle.createCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: [])
+      try checkCASForResults(jobs: jobs, cas: cas, fs: driver.fileSystem)
+
+      // try replan the job and make sure some key command-line options are generated.
+      let rebuildJobs = try driver.planBuild()
+      for job in rebuildJobs {
+        if job.kind == .compile || job.kind == .emitModule {
+          XCTAssertTrue(job.commandLine.contains(.flag(String("-disable-implicit-swift-modules"))))
+          XCTAssertTrue(job.commandLine.contains(.flag(String("-cache-compile-job"))))
+          XCTAssertTrue(job.commandLine.contains(.flag(String("-cas-path"))))
+          XCTAssertTrue(job.commandLine.contains(.flag(String("-bridging-header-pch-key"))))
+        }
+      }
+    }
+  }
 }