Merge pull request #1853 from cachemeifyoucan/eng/PR-147874145

[Incremental] Track and return skipped non-compile job

Author: cachemeifyoucan

Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift

@@ -51,10 +51,11 @@ extension IncrementalCompilationState {
 
     public func compute(batchJobFormer: inout Driver) throws -> FirstWave {
       return try blockingConcurrentAccessOrMutation {
-        let (initiallySkippedCompileJobs, mandatoryJobsInOrder, afterCompiles) =
+        let (initiallySkippedCompileJobs, skippedNonCompileJobs, mandatoryJobsInOrder, afterCompiles) =
         try computeInputsAndGroups(batchJobFormer: &batchJobFormer)
         return FirstWave(
           initiallySkippedCompileJobs: initiallySkippedCompileJobs,
+          skippedNonCompileJobs: skippedNonCompileJobs,
           mandatoryJobsInOrder: mandatoryJobsInOrder,
           jobsAfterCompiles: afterCompiles)
       }
@@ -75,6 +76,7 @@ extension IncrementalCompilationState.FirstWaveComputer {
   /// listed in fingerprintExternalDependencies.
   private func computeInputsAndGroups(batchJobFormer: inout Driver)
   throws -> (initiallySkippedCompileJobs: [TypedVirtualPath: Job],
+             skippedNonCompileJobs: [Job],
              mandatoryJobsInOrder: [Job],
              jobsAfterCompiles: [Job])
   {
@@ -86,6 +88,7 @@ extension IncrementalCompilationState.FirstWaveComputer {
 
     func everythingIsMandatory()
       throws -> (initiallySkippedCompileJobs: [TypedVirtualPath: Job],
+                 skippedNonCompileJobs: [Job],
                  mandatoryJobsInOrder: [Job],
                  jobsAfterCompiles: [Job])
     {
@@ -103,6 +106,7 @@ extension IncrementalCompilationState.FirstWaveComputer {
 
       moduleDependencyGraph.setPhase(to: .buildingAfterEachCompilation)
       return (initiallySkippedCompileJobs: [:],
+              skippedNonCompileJobs: [],
               mandatoryJobsInOrder: mandatoryJobsInOrder,
               jobsAfterCompiles: jobsInPhases.afterCompiles)
     }
@@ -133,14 +137,20 @@ extension IncrementalCompilationState.FirstWaveComputer {
     // we can skip running `beforeCompiles` jobs if we also ensure that none of the `afterCompiles` jobs
     // have any dependencies on them.
     let skipAllJobs = batchedCompilationJobs.isEmpty ? !nonVerifyAfterCompileJobsDependOnBeforeCompileJobs() : false
+    let beforeCompileJobs = skipAllJobs ? [] : jobsInPhases.beforeCompiles
+    var skippedNonCompileJobs = skipAllJobs ? jobsInPhases.beforeCompiles : []
 
     // Schedule emitModule job together with verify module interface job.
-    let beforeCompileJobs = skipAllJobs ? [] : jobsInPhases.beforeCompiles
-    let afterCompileJobs = jobsInPhases.afterCompiles.compactMap { job in
-      skipAllJobs && job.kind == .verifyModuleInterface ? nil : job
+    let afterCompileJobs = jobsInPhases.afterCompiles.compactMap { job -> Job? in
+      if skipAllJobs && job.kind == .verifyModuleInterface {
+        skippedNonCompileJobs.append(job)
+        return nil
+      }
+      return job
     }
     let mandatoryJobsInOrder = beforeCompileJobs + batchedCompilationJobs
     return (initiallySkippedCompileJobs: initiallySkippedCompileJobs,
+            skippedNonCompileJobs: skippedNonCompileJobs,
             mandatoryJobsInOrder: mandatoryJobsInOrder,
             jobsAfterCompiles: afterCompileJobs)
   }

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift

@@ -65,11 +65,14 @@ extension IncrementalCompilationState {
     /// incremental dependency graph and the status of the input files for this
     /// incremental build.
     let initiallySkippedCompileJobs: [TypedVirtualPath: Job]
+    /// The non-compile jobs that can be skipped given the state of the
+    /// incremental build.
+    let skippedNonCompileJobs: [Job]
     /// All of the pre-compile or compilation job (groups) known to be required
     /// for the first wave to execute.
     /// The primaries could be other than .swift files, i.e. .sib
     let mandatoryJobsInOrder: [Job]
-
+    /// The job after compilation that needs to run.
     let jobsAfterCompiles: [Job]
   }
 }

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift

@@ -44,6 +44,9 @@ public final class IncrementalCompilationState {
   /// Jobs to run *after* the last compile, for instance, link-editing.
   public let jobsAfterCompiles: [Job]
 
+  /// The skipped non compile jobs.
+  public let skippedJobsNonCompile: [Job]
+
   public let info: IncrementalCompilationState.IncrementalDependencyAndInputSetup
 
   internal let upToDateInterModuleDependencyGraph: InterModuleDependencyGraph?
@@ -78,6 +81,7 @@ public final class IncrementalCompilationState {
       &driver)
     self.mandatoryJobsInOrder = firstWave.mandatoryJobsInOrder
     self.jobsAfterCompiles = firstWave.jobsAfterCompiles
+    self.skippedJobsNonCompile = firstWave.skippedNonCompileJobs
   }
 
   /// Allow concurrent access to while preventing mutation of ``IncrementalCompilationState/protectedState``

Sources/SwiftDriver/Jobs/Planning.swift

@@ -94,7 +94,7 @@ extension Driver {
     // If the jobs are batched during the incremental build, reuse the computation rather than computing the batches again.
     if let incrementalState = incrementalCompilationState {
       // For compatibility reasons, all the jobs planned will be returned, even the incremental state suggests the job is not mandatory.
-      batchedJobs = incrementalState.skippedJobs + incrementalState.mandatoryJobsInOrder + incrementalState.jobsAfterCompiles
+      batchedJobs = incrementalState.skippedJobs + incrementalState.skippedJobsNonCompile + incrementalState.mandatoryJobsInOrder + incrementalState.jobsAfterCompiles
     } else {
       batchedJobs = try formBatchedJobs(jobsInPhases.allJobs,
                                         showJobLifecycle: showJobLifecycle,

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -845,15 +845,19 @@ final class ExplicitModuleBuildTests: XCTestCase {
       let incrementalJobs = try incrementalDriver.planBuild()
       try incrementalDriver.run(jobs: incrementalJobs)
       XCTAssertFalse(incrementalDriver.diagnosticEngine.hasErrors)
+      let state = try XCTUnwrap(incrementalDriver.incrementalCompilationState)
+      XCTAssertTrue(state.mandatoryJobsInOrder.contains { $0.kind == .emitModule })
+      XCTAssertTrue(state.jobsAfterCompiles.contains { $0.kind == .verifyModuleInterface })
 
       // TODO: emitModule job should run again if interface is deleted.
       // try localFileSystem.removeFileTree(swiftInterfaceOutput)
 
       // This should be a null build but it is actually building the main module due to the previous build of all the modules.
       var reDriver = try Driver(args: invocationArguments + ["-color-diagnostics"])
-      let reJobs = try reDriver.planBuild()
-      XCTAssertFalse(reJobs.contains { $0.kind == .emitModule })
-      XCTAssertFalse(reJobs.contains { $0.kind == .verifyModuleInterface })
+      let _ = try reDriver.planBuild()
+      let reState = try XCTUnwrap(reDriver.incrementalCompilationState)
+      XCTAssertFalse(reState.mandatoryJobsInOrder.contains { $0.kind == .emitModule })
+      XCTAssertFalse(reState.jobsAfterCompiles.contains { $0.kind == .verifyModuleInterface })
     }
   }
 

Tests/SwiftDriverTests/IncrementalCompilationTests.swift

@@ -220,12 +220,14 @@ extension IncrementalCompilationTests {
 
   // Null planning should not return an empty compile job for compatibility reason.
   // `swift-build` wraps the jobs returned by swift-driver in `Executor` so returning an empty list of compile job will break build system.
-  func testNullPlanningCompatility() throws {
+  func testNullPlanningCompatibility() throws {
     guard let sdkArgumentsForTesting = try Driver.sdkArgumentsForTesting() else {
       throw XCTSkip("Cannot perform this test on this host")
     }
-    var driver = try Driver(args: commonArgs + sdkArgumentsForTesting)
+    let extraArguments = ["-experimental-emit-module-separately", "-emit-module"]
+    var driver = try Driver(args: commonArgs + extraArguments + sdkArgumentsForTesting)
     let initialJobs = try driver.planBuild()
+    XCTAssertTrue(initialJobs.contains { $0.kind == .emitModule})
     try driver.run(jobs: initialJobs)
 
     // Plan the build again without touching any file. This should be a null build but for compatibility reason,
@@ -235,6 +237,7 @@ extension IncrementalCompilationTests {
     XCTAssertFalse(
       replanJobs.filter { $0.kind == .compile }.isEmpty,
       "more than one compile job needs to be planned")
+    XCTAssertTrue(replanJobs.contains { $0.kind == .emitModule})
   }
 }