[Caching] Invalidate built module if missing from CAS

For swift caching build, swift-frontend only reads from CAS for inputs
for full build reproducibility. In this case, if the CAS is removed, we
need to treat the modules as missing from CAS and rebuild them, even if
the version on disk appears to pass the up-to-date check.

rdar://135718227

Author: cachemeifyoucan

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift

@@ -158,6 +158,7 @@ internal extension InterModuleDependencyGraph {
   /// between it and the root (source module being built by this driver
   /// instance) must also be re-built.
   func computeInvalidatedModuleDependencies(fileSystem: FileSystem,
+                                            cas: SwiftScanCAS?,
                                             forRebuild: Bool,
                                             reporter: IncrementalCompilationState.Reporter? = nil)
   throws -> Set<ModuleDependencyId> {
@@ -169,7 +170,7 @@ internal extension InterModuleDependencyGraph {
     for dependencyId in mainModuleInfo.directDependencies ?? [] {
       try outOfDateModuleScan(from: dependencyId, visited: &visited,
                               modulesRequiringRebuild: &modulesRequiringRebuild,
-                              fileSystem: fileSystem, forRebuild: forRebuild,
+                              fileSystem: fileSystem, cas: cas, forRebuild: forRebuild,
                               reporter: reporter)
     }
 
@@ -183,10 +184,11 @@ internal extension InterModuleDependencyGraph {
   /// filter out those with a fully up-to-date output
   func filterMandatoryModuleDependencyCompileJobs(_ allJobs: [Job],
                                                   fileSystem: FileSystem,
+                                                  cas: SwiftScanCAS?,
                                                   reporter: IncrementalCompilationState.Reporter? = nil) throws -> [Job] {
     // Determine which module pre-build jobs must be re-run
     let modulesRequiringReBuild =
-      try computeInvalidatedModuleDependencies(fileSystem: fileSystem, forRebuild: true, reporter: reporter)
+      try computeInvalidatedModuleDependencies(fileSystem: fileSystem, cas: cas, forRebuild: true, reporter: reporter)
 
     // Filter the `.generatePCM` and `.compileModuleFromInterface` jobs for
     // modules which do *not* need re-building.
@@ -209,6 +211,7 @@ internal extension InterModuleDependencyGraph {
                            visited: inout Set<ModuleDependencyId>,
                            modulesRequiringRebuild: inout Set<ModuleDependencyId>,
                            fileSystem: FileSystem,
+                           cas: SwiftScanCAS?,
                            forRebuild: Bool,
                            reporter: IncrementalCompilationState.Reporter? = nil) throws {
     let reportOutOfDate = { (name: String, reason: String)  in
@@ -227,7 +230,7 @@ internal extension InterModuleDependencyGraph {
       if !visited.contains(dependencyId) {
         try outOfDateModuleScan(from: dependencyId, visited: &visited,
                                 modulesRequiringRebuild: &modulesRequiringRebuild,
-                                fileSystem: fileSystem, forRebuild: forRebuild,
+                                fileSystem: fileSystem, cas: cas, forRebuild: forRebuild,
                                 reporter: reporter)
       }
       // Even if we're not revisiting a dependency, we must check if it's already known to be out of date.
@@ -237,7 +240,7 @@ internal extension InterModuleDependencyGraph {
     if hasOutOfDateModuleDependency {
       reportOutOfDate(sourceModuleId.moduleNameForDiagnostic, "Invalidated by downstream dependency")
       modulesRequiringRebuild.insert(sourceModuleId)
-    } else if try !verifyModuleDependencyUpToDate(moduleID: sourceModuleId, fileSystem: fileSystem, reporter: reporter) {
+    } else if try !verifyModuleDependencyUpToDate(moduleID: sourceModuleId, fileSystem: fileSystem, cas:cas, reporter: reporter) {
       reportOutOfDate(sourceModuleId.moduleNameForDiagnostic, "Out-of-date")
       modulesRequiringRebuild.insert(sourceModuleId)
     }
@@ -246,10 +249,44 @@ internal extension InterModuleDependencyGraph {
     visited.insert(sourceModuleId)
   }
 
+  func outputMissingFromCAS(moduleInfo: ModuleInfo,
+                            cas: SwiftScanCAS?) throws -> Bool {
+    func casOutputMissing(_ key: String?) throws -> Bool {
+      // Caching not enabled.
+      guard let id = key, let cas = cas else { return false }
+      // Do a local query to see if the output exists.
+      let result = try cas.queryCacheKey(id, globally: false)
+      // Make sure all outputs are available in local CAS.
+      guard let outputs = result else { return true }
+      return !outputs.allSatisfy { $0.isMaterialized }
+    }
+
+    switch moduleInfo.details {
+    case .swift(let swiftDetails):
+      return try casOutputMissing(swiftDetails.moduleCacheKey)
+    case .clang(let clangDetails):
+      return try casOutputMissing(clangDetails.moduleCacheKey)
+    case .swiftPrebuiltExternal(_):
+      return false;
+    case .swiftPlaceholder(_):
+      // TODO: This should never ever happen. Hard error?
+      return true;
+    }
+  }
+
   func verifyModuleDependencyUpToDate(moduleID: ModuleDependencyId,
                                       fileSystem: FileSystem,
+                                      cas: SwiftScanCAS?,
                                       reporter: IncrementalCompilationState.Reporter?) throws -> Bool {
     let checkedModuleInfo = try moduleInfo(of: moduleID)
+    // Check if there is a module cache key available, then the content that pointed by the cache key must
+    // exist for module to be up-to-date. Treat any CAS error as missing.
+    let missingFromCAS = (try? outputMissingFromCAS(moduleInfo: checkedModuleInfo, cas: cas)) ?? true
+    if missingFromCAS {
+      reporter?.reportExplicitDependencyMissingFromCAS(moduleID.moduleName)
+      return false
+    }
+
     // Verify that the specified input exists and is older than the specified output
     let verifyInputOlderThanOutputModTime: (String, VirtualPath, TimePoint) -> Bool =
     { moduleName, inputPath, outputModTime in

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift

@@ -317,6 +317,10 @@ extension IncrementalCompilationState {
       report("Following explicit module dependencies will be re-built: [\(modules.map { $0.moduleNameForDiagnostic }.sorted().joined(separator: ", "))]")
     }
 
+    func reportExplicitDependencyMissingFromCAS(_ moduleName: String) {
+      report("Dependency module \(moduleName) is missing from CAS")
+    }
+
     // Emits a remark indicating incremental compilation has been disabled.
     func reportDisablingIncrementalBuild(_ why: String) {
       report("Disabling incremental build: \(why)")

Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift

@@ -120,6 +120,7 @@ extension IncrementalCompilationState.IncrementalDependencyAndInputSetup {
 
     // Verify that each dependnecy is up-to-date with respect to its inputs
     guard try priorInterModuleDependencyGraph.computeInvalidatedModuleDependencies(fileSystem: buildRecordInfo.fileSystem,
+                                                                                   cas: driver.cas,
                                                                                    forRebuild: false,
                                                                                    reporter: reporter).isEmpty else {
       reporter?.reportExplicitBuildMustReScan("Not all dependencies are up-to-date.")

Sources/SwiftDriver/Jobs/Planning.swift

@@ -236,6 +236,7 @@ extension Driver {
       mandatoryModuleCompileJobs =
         try resolvedDependencyGraph.filterMandatoryModuleDependencyCompileJobs(modulePrebuildJobs,
                                                                                fileSystem: fileSystem,
+                                                                               cas: cas,
                                                                                reporter: reporter)
     }
     mandatoryModuleCompileJobs.forEach(addJob)

TestInputs/ExplicitModuleBuilds/Swift/O.swiftinterface

@@ -0,0 +1,3 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name O -parse-stdlib
+public func none() { }

Tests/SwiftDriverTests/IncrementalCompilationTests.swift

@@ -49,6 +49,9 @@ final class IncrementalCompilationTests: XCTestCase {
   var priorsPath: AbsolutePath {
     derivedDataPath.appending(component: "\(module)-master.priors")
   }
+  var casPath: AbsolutePath {
+    derivedDataPath.appending(component: "cas")
+  }
   func swiftDepsPath(basename: String) -> AbsolutePath {
     derivedDataPath.appending(component: "\(basename).swiftdeps")
   }
@@ -594,6 +597,59 @@ extension IncrementalCompilationTests {
   }
 }
 
+// MARK: - Explicit compilation caching incremental tests
+extension IncrementalCompilationTests {
+  func testIncrementalCompilationCaching() throws {
+#if os(Windows)
+    throw XCTSkip("caching not supported on windows")
+#else
+    let driver = try Driver(args: ["swiftc"])
+    guard driver.isFeatureSupported(.compilation_caching) else {
+      throw XCTSkip("caching not supported")
+    }
+#endif
+    let extraArguments = ["-cache-compile-job", "-cas-path", casPath.nativePathString(escaped: true), "-O", "-parse-stdlib"]
+    replace(contentsOf: "other", with: "import O;")
+    // Simplified initial build.
+    try doABuild(
+      "Initial Simplified Build with Caching",
+      checkDiagnostics: false,
+      extraArguments: explicitBuildArgs + extraArguments,
+      whenAutolinking: autolinkLifecycleExpectedDiags) {
+      startCompilingExplicitSwiftDependency("O")
+      finishCompilingExplicitSwiftDependency("O")
+      compiling("main", "other")
+    }
+
+    // Delete the CAS, touch a file then rebuild.
+    try localFileSystem.removeFileTree(casPath)
+
+    // Deleting the CAS should cause a full rebuild since all modules are missing from CAS.
+    try doABuild(
+      "Deleting CAS and rebuild",
+      checkDiagnostics: false,
+      extraArguments: explicitBuildArgs + extraArguments,
+      whenAutolinking: autolinkLifecycleExpectedDiags
+    ) {
+      readGraph
+      readInterModuleGraph
+      explicitDependencyModuleMissingFromCAS("O")
+      moduleInfoStaleOutOfDate("O")
+      explicitMustReScanDueToChangedDependencyInput
+      moduleWillBeRebuiltOutOfDate("O")
+      explicitModulesWillBeRebuilt(["O"])
+      compilingExplicitSwiftDependency("O")
+      foundBatchableJobs(2)
+      formingOneBatch
+      addingToBatchThenForming("main", "other")
+      startCompilingExplicitSwiftDependency("O")
+      finishCompilingExplicitSwiftDependency("O")
+      compiling("main", "other")
+    }
+  }
+}
+
+
 // MARK: - Simpler incremental tests
 extension IncrementalCompilationTests {
 
@@ -1764,6 +1820,9 @@ extension DiagVerifiable {
   @DiagsBuilder func explicitModulesWillBeRebuilt(_ moduleNames: [String]) -> [Diagnostic.Message] {
     "Incremental compilation: Following explicit module dependencies will be re-built: [\(moduleNames.joined(separator: ", "))]"
   }
+  @DiagsBuilder func explicitDependencyModuleMissingFromCAS(_ dependencyModuleName: String) -> [Diagnostic.Message] {
+    "Dependency module \(dependencyModuleName) is missing from CAS"
+  }
 
   // MARK: - misc
   @DiagsBuilder func disabledForRemoval(_ removedInput: String) -> [Diagnostic.Message] {