Merge pull request #1491 from cachemeifyoucan/eng/PR-swift-caching-build-system-API-refine

[Caching] SwiftDriver Caching API refinement for build system

Author: cachemeifyoucan

Sources/CSwiftScan/include/swiftscan_header.h

@@ -332,6 +332,11 @@ typedef struct {
   void (*swiftscan_cache_cancellation_token_dispose)(
       swiftscan_cache_cancellation_token_t);
 
+  void (*swiftscan_cache_download_cas_object_async)(
+      swiftscan_cas_t, const char *id, void *ctx,
+      void (*callback)(void *ctx, bool success, swiftscan_string_ref_t error),
+      swiftscan_cache_cancellation_token_t *);
+
   swiftscan_cache_replay_instance_t (*swiftscan_cache_replay_instance_create)(
       int argc, const char **argv, swiftscan_string_ref_t *error);
   void (*swiftscan_cache_replay_instance_dispose)(

Sources/SwiftDriver/Driver/Driver.swift

@@ -275,7 +275,7 @@ public struct Driver {
   let useClangIncludeTree: Bool
 
   /// CAS instance used for compilation.
-  public var cas: SwiftScanCAS? = nil
+  var cas: SwiftScanCAS? = nil
 
   /// Is swift caching enabled.
   lazy var isCachingEnabled: Bool = {

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift

@@ -165,7 +165,7 @@ public class InterModuleDependencyOracle {
     return diags.isEmpty ? nil : diags
   }
 
-  @_spi(Testing) public func createCAS(pluginPath: AbsolutePath?, onDiskPath: AbsolutePath?, pluginOptions: [(String, String)]) throws -> SwiftScanCAS {
+  public func createCAS(pluginPath: AbsolutePath?, onDiskPath: AbsolutePath?, pluginOptions: [(String, String)]) throws -> SwiftScanCAS {
     guard let swiftScan = swiftScanLibInstance else {
       fatalError("Attempting to reset scanner cache with no scanner instance.")
     }

Sources/SwiftDriver/SwiftScan/SwiftScan.swift

@@ -528,6 +528,8 @@ private extension swiftscan_functions_t {
     self.swiftscan_cache_action_cancel = try loadOptional("swiftscan_cache_action_cancel")
     self.swiftscan_cache_cancellation_token_dispose = try loadOptional("swiftscan_cache_cancellation_token_dispose")
 
+    self.swiftscan_cache_download_cas_object_async = try loadOptional("swiftscan_cache_download_cas_object_async")
+
     self.swiftscan_cache_replay_instance_create = try loadOptional("swiftscan_cache_replay_instance_create")
     self.swiftscan_cache_replay_instance_dispose = try loadOptional("swiftscan_cache_replay_instance_dispose")
     self.swiftscan_cache_replay_compilation = try loadOptional("swiftscan_cache_replay_compilation")

Sources/SwiftDriver/SwiftScan/SwiftScanCAS.swift

@@ -37,6 +37,36 @@ public final class CachedCompilation {
     lib.api.swiftscan_cached_compilation_is_uncacheable(ptr)
   }
 
+  public func makeGlobal(_ callback: @escaping (Swift.Error?) -> ()) {
+    class CallbackContext {
+      func retain() -> UnsafeMutableRawPointer {
+        return Unmanaged.passRetained(self).toOpaque()
+      }
+
+      let comp: CachedCompilation
+      let callback: (Swift.Error?) -> ()
+      init(_ compilation: CachedCompilation, _ callback: @escaping (Swift.Error?) -> ()) {
+        self.comp = compilation
+        self.callback = callback
+      }
+    }
+
+    func callbackFunc(_ context: UnsafeMutableRawPointer?, _ error: swiftscan_string_ref_t) {
+      let obj = Unmanaged<CallbackContext>.fromOpaque(context!).takeRetainedValue()
+      if error.length != 0 {
+        if let err = try? obj.comp.lib.toSwiftString(error) {
+          obj.callback(DependencyScanningError.casError(err))
+        } else {
+          obj.callback(DependencyScanningError.casError("unknown makeGlobal error"))
+        }
+      }
+      obj.callback(nil)
+    }
+
+    let context = CallbackContext(self, callback)
+    lib.api.swiftscan_cached_compilation_make_global_async(ptr, context.retain(), callbackFunc, nil)
+  }
+
   deinit {
     lib.api.swiftscan_cached_compilation_dispose(ptr)
   }
@@ -235,35 +265,15 @@ extension swiftscan_cache_replay_result_t {
 #if canImport(_Concurrency)
 // Async API Vendor
 extension CachedCompilation {
-  public func makeGlobal() async throws -> Bool {
-    class CallbackContext {
-      func retain() -> UnsafeMutableRawPointer {
-        return Unmanaged.passRetained(self).toOpaque()
-      }
-
-      let continuation: CheckedContinuation<Bool, Swift.Error>
-      let comp: CachedCompilation
-      init(_ continuation: CheckedContinuation<Bool, Swift.Error>, compilation: CachedCompilation) {
-        self.continuation = continuation
-        self.comp = compilation
-      }
-    }
-
-    func callbackFunc(_ context: UnsafeMutableRawPointer?, _ error: swiftscan_string_ref_t) {
-      let obj = Unmanaged<CallbackContext>.fromOpaque(context!).takeRetainedValue()
-      if error.length != 0 {
-        if let err = try? obj.comp.lib.toSwiftString(error) {
-          obj.continuation.resume(throwing: DependencyScanningError.casError(err))
+  public func makeGlobal() async throws {
+    return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Swift.Error>) in
+      makeGlobal { (error: Swift.Error?) in
+        if let err = error {
+          continuation.resume(throwing: err)
         } else {
-          obj.continuation.resume(throwing: DependencyScanningError.casError("unknown makeGlobal error"))
+          continuation.resume(returning: ())
         }
       }
-      obj.continuation.resume(returning: true)
-    }
-
-    return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Bool, Swift.Error>) in
-      let context = CallbackContext(continuation, compilation: self)
-      lib.api.swiftscan_cached_compilation_make_global_async(ptr, context.retain(), callbackFunc, nil)
     }
   }
 }
@@ -334,5 +344,37 @@ extension SwiftScanCAS {
       scanner.api.swiftscan_cache_query_async(cas, key.cString(using: .utf8), globally, context.retain(), callbackFunc, nil)
     }
   }
+
+  public func download(with id: String) async throws -> Bool {
+    class CallbackContext {
+      func retain() -> UnsafeMutableRawPointer {
+        return Unmanaged.passRetained(self).toOpaque()
+      }
+
+      let continuation: CheckedContinuation<Bool, Swift.Error>
+      let cas: SwiftScanCAS
+      init(_ continuation: CheckedContinuation<Bool, Swift.Error>, cas: SwiftScanCAS) {
+        self.continuation = continuation
+        self.cas = cas
+      }
+    }
+
+    func callbackFunc(_ context: UnsafeMutableRawPointer?, _ success: Bool, _ error: swiftscan_string_ref_t) {
+      let obj = Unmanaged<CallbackContext>.fromOpaque(context!).takeRetainedValue()
+      if error.length != 0 {
+        if let err = try? obj.cas.scanner.toSwiftString(error) {
+          obj.continuation.resume(throwing: DependencyScanningError.casError(err))
+        } else {
+          obj.continuation.resume(throwing: DependencyScanningError.casError("unknown output loading error"))
+        }
+      }
+      obj.continuation.resume(returning: success)
+    }
+
+    return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Bool, Swift.Error>) in
+      let context = CallbackContext(continuation, cas: self)
+      scanner.api.swiftscan_cache_download_cas_object_async(cas, id.cString(using: .utf8), context.retain(), callbackFunc, nil)
+    }
+  }
 }
 #endif

Tests/SwiftDriverTests/CachingBuildTests.swift

@@ -119,6 +119,17 @@ private func checkCASForResults(jobs: [Job], cas: SwiftScanCAS, fs: FileSystem)
             XCTAssertTrue(output.isMaterialized, "Cached output not founded in CAS")
             let success = try await output.load()
             XCTAssertTrue(success, "Cached output not founded in CAS")
+
+            // Try async download. Download should succeed even on a local CAS.
+            let casID = try output.getCASID()
+            let downloaded = try await cas.download(with: casID)
+            XCTAssertTrue(downloaded, "Cached output cannot be downloaded")
+          }
+          // Execise the uploading path.
+          try await compilation.makeGlobal()
+          // Execise call back uploading method.
+          compilation.makeGlobal { error in
+            XCTAssertTrue(error == nil, "Upload Error")
           }
           compilations.append(compilation)
         } else {
@@ -487,10 +498,17 @@ final class CachingBuildTests: XCTestCase {
       let jobs = try driver.planBuild()
       try driver.run(jobs: jobs)
       XCTAssertFalse(driver.diagnosticEngine.hasErrors)
-      guard let cas = driver.cas else {
-        XCTFail("CAS is not available")
+
+      let dependencyOracle = InterModuleDependencyOracle()
+      let scanLibPath = try XCTUnwrap(driver.toolchain.lookupSwiftScanLib())
+      guard try dependencyOracle
+              .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
+                                             swiftScanLibPath: scanLibPath) else {
+        XCTFail("Dependency scanner library not found")
         return
       }
+
+      let cas = try dependencyOracle.createCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: [])
       try checkCASForResults(jobs: jobs, cas: cas, fs: driver.fileSystem)
     }
   }
@@ -555,6 +573,9 @@ final class CachingBuildTests: XCTestCase {
       try fooBuildDriver.run(jobs: fooJobs)
       XCTAssertFalse(fooBuildDriver.diagnosticEngine.hasErrors)
 
+      let cas = try dependencyOracle.createCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: [])
+      try checkCASForResults(jobs: fooJobs, cas: cas, fs: fooBuildDriver.fileSystem)
+
       var driver = try Driver(args: ["swiftc",
                                      "-I", FooInstallPath.nativePathString(escaped: true),
                                      "-explicit-module-build", "-emit-module", "-emit-module-path",