Make AsyncHTTPClient optional dependency (#101)

* Add a new abstraction `HTTPClientProtocol` to abstract the HTTP client
  operations.
* Add a new `OfflineHTTPClient` type that just throws an error when
  trying to download a file.
* Remove `httpClient` member from `Engine` and pass the HTTP client as a
  parameter to the queries that need it.
* Separate `Query` and `CacheKey` protocols to allow for queries having
  properties that are not part of the cache key.

Author: kateinoigakukun

Package.swift

@@ -15,7 +15,6 @@ let package = Package(
   ],
   dependencies: [
     // Dependencies declare other packages that this package depends on.
-    .package(url: "https://github.com/swift-server/async-http-client.git", from: "1.19.0"),
     .package(url: "https://github.com/apple/swift-system", from: "1.2.1"),
     .package(url: "https://github.com/apple/swift-argument-parser", from: "1.2.2"),
     .package(url: "https://github.com/apple/swift-async-algorithms.git", exact: "1.0.0-beta.1"),
@@ -43,7 +42,7 @@ let package = Package(
       dependencies: [
         .target(name: "AsyncProcess"),
         .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"),
-        .product(name: "AsyncHTTPClient", package: "async-http-client"),
+        .product(name: "NIOHTTP1", package: "swift-nio"),
         .product(name: "Logging", package: "swift-log"),
         .product(name: "SystemPackage", package: "swift-system"),
         "GeneratorEngine",
@@ -66,7 +65,6 @@ let package = Package(
     .target(
       name: "GeneratorEngine",
       dependencies: [
-        .product(name: "AsyncHTTPClient", package: "async-http-client"),
         .product(name: "Crypto", package: "swift-crypto"),
         .product(name: "Logging", package: "swift-log"),
         .product(name: "SystemPackage", package: "swift-system"),
@@ -118,3 +116,26 @@ let package = Package(
     ),
   ]
 )
+
+struct Configuration {
+  let useAsyncHttpClient: Bool
+  init(SWIFT_SDK_GENERATOR_DISABLE_AHC: Bool) {
+    self.useAsyncHttpClient = !SWIFT_SDK_GENERATOR_DISABLE_AHC
+  }
+}
+
+let configuration = Configuration(
+  SWIFT_SDK_GENERATOR_DISABLE_AHC: Context.environment["SWIFT_SDK_GENERATOR_DISABLE_AHC"] != nil
+)
+
+if configuration.useAsyncHttpClient {
+  package.dependencies.append(
+    .package(url: "https://github.com/swift-server/async-http-client.git", from: "1.19.0")
+  )
+  let targetsToAppend: Set<String> = ["SwiftSDKGenerator"]
+  for target in package.targets.filter({ targetsToAppend.contains($0.name) }) {
+    target.dependencies.append(
+      .product(name: "AsyncHTTPClient", package: "async-http-client")
+    )
+  }
+}

Sources/GeneratorEngine/Engine.swift

@@ -10,7 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-import class AsyncHTTPClient.HTTPClient
 @_exported import Crypto
 import struct Logging.Logger
 @_exported import struct SystemPackage.FilePath
@@ -42,7 +41,6 @@ public actor Engine {
     private(set) var cacheMisses = 0
 
     public let fileSystem: any FileSystem
-    public let httpClient = HTTPClient()
     public let logger: Logger
     private let resultsCache: SQLiteBackedCache
     private var isShutDown = false
@@ -66,7 +64,6 @@ public actor Engine {
     public func shutDown() async throws {
         precondition(!self.isShutDown, "`QueryEngine/shutDown` should be called only once")
         try self.resultsCache.close()
-        try await self.httpClient.shutdown()
 
         self.isShutDown = true
     }
@@ -85,7 +82,7 @@ public actor Engine {
     public subscript(_ query: some Query) -> FileCacheRecord {
         get async throws {
             let hashEncoder = HashEncoder<SHA512>()
-            try hashEncoder.encode(query)
+            try hashEncoder.encode(query.cacheKey)
             let key = hashEncoder.finalize()
 
             if let fileRecord = try resultsCache.get(key, as: FileCacheRecord.self) {

Sources/GeneratorEngine/Query.swift

@@ -12,10 +12,17 @@
 
 import struct SystemPackage.FilePath
 
-public protocol Query: CacheKey, Sendable {
+public protocol Query: Sendable {
+  associatedtype Key: CacheKey
+  var cacheKey: Key { get }
   func run(engine: Engine) async throws -> FilePath
 }
 
+public protocol CachingQuery: Query, CacheKey where Self.Key == Self {}
+extension CachingQuery {
+  public var cacheKey: Key { self }
+}
+
 final class HashEncoder<Hash: HashFunction>: Encoder {
     enum Error: Swift.Error {
         case noCacheKeyConformance(Encodable.Type)

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift

@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 import AsyncAlgorithms
-import AsyncHTTPClient
 import GeneratorEngine
 import RegexBuilder
 
@@ -28,16 +27,19 @@ let byteCountFormatter = ByteCountFormatter()
 
 extension SwiftSDKGenerator {
   func downloadArtifacts(
-    _ client: HTTPClient, _ engine: Engine,
+    _ client: some HTTPClientProtocol, _ engine: Engine,
     downloadableArtifacts: inout DownloadableArtifacts,
     itemsToDownload: @Sendable (DownloadableArtifacts) -> [DownloadableArtifacts.Item]
   ) async throws {
     logGenerationStep("Downloading required toolchain packages...")
-    var headRequest = HTTPClientRequest(url: downloadableArtifacts.hostLLVM.remoteURL.absoluteString)
-    headRequest.method = .HEAD
-    headRequest.headers = ["Accept": "*/*", "User-Agent": "Swift SDK Generator"]
-    let isLLVMBinaryArtifactAvailable = try await client.execute(headRequest, deadline: .distantFuture)
-      .status == .ok
+    let hostLLVMURL = downloadableArtifacts.hostLLVM.remoteURL
+    // Workaround an issue with github.com returning 400 instead of 404 status to HEAD requests from AHC.
+    let isLLVMBinaryArtifactAvailable = try await type(of: client).with(http1Only: true) {
+      try await $0.head(
+        url: hostLLVMURL.absoluteString,
+        headers: ["Accept": "*/*", "User-Agent": "Swift SDK Generator"]
+      )
+    }
 
     if !isLLVMBinaryArtifactAvailable {
       downloadableArtifacts.useLLVMSources()
@@ -46,7 +48,7 @@ extension SwiftSDKGenerator {
     let results = try await withThrowingTaskGroup(of: FileCacheRecord.self) { group in
       for item in itemsToDownload(downloadableArtifacts) {
         group.addTask {
-          try await engine[DownloadArtifactQuery(artifact: item)]
+          try await engine[DownloadArtifactQuery(artifact: item, httpClient: client)]
         }
       }
 
@@ -64,7 +66,7 @@ extension SwiftSDKGenerator {
   }
 
   func downloadUbuntuPackages(
-    _ client: HTTPClient,
+    _ client: some HTTPClientProtocol,
     _ engine: Engine,
     requiredPackages: [String],
     versionsConfiguration: VersionsConfiguration,
@@ -112,7 +114,7 @@ extension SwiftSDKGenerator {
     let pathsConfiguration = self.pathsConfiguration
 
     try await inTemporaryDirectory { fs, tmpDir in
-      let downloadedFiles = try await self.downloadFiles(from: urls, to: tmpDir, engine)
+      let downloadedFiles = try await self.downloadFiles(from: urls, to: tmpDir, client, engine)
       report(downloadedFiles: downloadedFiles)
 
       for fileName in urls.map(\.lastPathComponent) {
@@ -123,11 +125,13 @@ extension SwiftSDKGenerator {
     try createDirectoryIfNeeded(at: pathsConfiguration.toolchainBinDirPath)
   }
 
-  func downloadFiles(from urls: [URL], to directory: FilePath, _ engine: Engine) async throws -> [(URL, UInt64)] {
+  func downloadFiles(from urls: [URL], to directory: FilePath, _ client: some HTTPClientProtocol, _ engine: Engine) async throws -> [(URL, UInt64)] {
     try await withThrowingTaskGroup(of: (URL, UInt64).self) {
       for url in urls {
         $0.addTask {
-          let downloadedFilePath = try await engine[DownloadFileQuery(remoteURL: url, localDirectory: directory)]
+          let downloadedFilePath = try await engine[DownloadFileQuery(
+            remoteURL: url, localDirectory: directory, httpClient: client
+          )]
           let filePath = downloadedFilePath.path
           guard let fileSize = try FileManager.default.attributesOfItem(
             atPath: filePath.string
@@ -153,12 +157,12 @@ private func report(downloadedFiles: [(URL, UInt64)]) {
   }
 }
 
-extension HTTPClient {
+extension HTTPClientProtocol {
   private func downloadUbuntuPackagesList(
     from url: String,
     isVerbose: Bool
   ) async throws -> String? {
-    guard let packages = try await get(url: url).get().body?.unzip(isVerbose: isVerbose) else {
+    guard let packages = try await get(url: url).body?.unzip(isVerbose: isVerbose) else {
       throw FileOperationError.downloadFailed(url)
     }
 

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift

@@ -11,7 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 import AsyncAlgorithms
+#if canImport(AsyncHTTPClient)
 import AsyncHTTPClient
+#endif
 import Foundation
 import GeneratorEngine
 import RegexBuilder
@@ -30,25 +32,16 @@ public extension Triple.Arch {
   }
 }
 
-private func withHTTPClient(
-  _ configuration: HTTPClient.Configuration,
-  _ body: @Sendable (HTTPClient) async throws -> ()
-) async throws {
-  let client = HTTPClient(eventLoopGroupProvider: .singleton, configuration: configuration)
-  try await withAsyncThrowing {
-    try await body(client)
-  } defer: {
-    try await client.shutdown()
-  }
-}
-
 extension SwiftSDKGenerator {
   public func run(recipe: SwiftSDKRecipe) async throws {
     try await withEngine(LocalFileSystem(), self.logger, cacheLocation: self.engineCachePath) { engine in
-      var configuration = HTTPClient.Configuration(redirectConfiguration: .follow(max: 5, allowCycles: false))
-      // Workaround an issue with github.com returning 400 instead of 404 status to HEAD requests from AHC.
-      configuration.httpVersion = .http1Only
-      try await withHTTPClient(configuration) { client in
+      let httpClientType: HTTPClientProtocol.Type
+      #if canImport(AsyncHTTPClient)
+      httpClientType = HTTPClient.self
+      #else
+      httpClientType = OfflineHTTPClient.self
+      #endif
+      try await httpClientType.with { client in
         if !self.isIncremental {
           try await self.removeRecursively(at: pathsConfiguration.toolchainDirPath)
         }

Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift

@@ -13,7 +13,7 @@
 import GeneratorEngine
 import struct SystemPackage.FilePath
 
-struct CMakeBuildQuery: Query {
+struct CMakeBuildQuery: CachingQuery {
   let sourcesDirectory: FilePath
   /// Path to the output binary relative to the CMake build directory.
   let outputBinarySubpath: [FilePath.Component]

Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift

@@ -10,21 +10,24 @@
 //
 //===----------------------------------------------------------------------===//
 
-import class AsyncHTTPClient.FileDownloadDelegate
 import GeneratorEngine
 import struct SystemPackage.FilePath
 
 struct DownloadArtifactQuery: Query {
+  var cacheKey: some CacheKey { artifact }
   let artifact: DownloadableArtifacts.Item
+  let httpClient: any HTTPClientProtocol
 
   func run(engine: Engine) async throws -> FilePath {
     print("Downloading remote artifact not available in local cache: \(self.artifact.remoteURL)")
-    let stream = await engine.httpClient.streamDownloadProgress(for: self.artifact)
+    let stream = httpClient.streamDownloadProgress(
+      from: self.artifact.remoteURL, to: self.artifact.localPath
+    )
       .removeDuplicates(by: didProgressChangeSignificantly)
       ._throttle(for: .seconds(1))
 
-    for try await item in stream {
-      report(progress: item.progress, for: item.artifact)
+    for try await progress in stream {
+      report(progress: progress, for: artifact)
     }
     return self.artifact.localPath
   }
@@ -39,17 +42,17 @@ struct DownloadArtifactQuery: Query {
 /// larger than 1MiB. Returns `false` otherwise.
 @Sendable
 private func didProgressChangeSignificantly(
-  previous: ArtifactDownloadProgress,
-  current: ArtifactDownloadProgress
+  previous: DownloadProgress,
+  current: DownloadProgress
 ) -> Bool {
-  guard previous.progress.totalBytes == current.progress.totalBytes else {
+  guard previous.totalBytes == current.totalBytes else {
     return true
   }
 
-  return current.progress.receivedBytes - previous.progress.receivedBytes > 1024 * 1024 * 1024
+  return current.receivedBytes - previous.receivedBytes > 1024 * 1024 * 1024
 }
 
-private func report(progress: FileDownloadDelegate.Progress, for artifact: DownloadableArtifacts.Item) {
+private func report(progress: DownloadProgress, for artifact: DownloadableArtifacts.Item) {
   if let total = progress.totalBytes {
     print("""
     \(artifact.remoteURL.lastPathComponent) \(

Sources/SwiftSDKGenerator/Queries/DownloadFileQuery.swift

@@ -15,12 +15,20 @@ import GeneratorEngine
 import struct SystemPackage.FilePath
 
 struct DownloadFileQuery: Query {
+  struct Key: CacheKey {
+    let remoteURL: URL
+    let localDirectory: FilePath
+  }
+  var cacheKey: Key {
+    Key(remoteURL: remoteURL, localDirectory: localDirectory)
+  }
   let remoteURL: URL
   let localDirectory: FilePath
+  let httpClient: any HTTPClientProtocol
 
   func run(engine: Engine) async throws -> FilePath {
     let downloadedFilePath = self.localDirectory.appending(self.remoteURL.lastPathComponent)
-    _ = try await engine.httpClient.downloadFile(from: self.remoteURL, to: downloadedFilePath)
+    _ = try await httpClient.downloadFile(from: self.remoteURL, to: downloadedFilePath)
     return downloadedFilePath
   }
 }

Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift

@@ -10,7 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-import AsyncHTTPClient
 import Foundation
 import GeneratorEngine
 import struct SystemPackage.FilePath
@@ -128,7 +127,7 @@ public struct LinuxRecipe: SwiftSDKRecipe {
   public func makeSwiftSDK(
     generator: SwiftSDKGenerator,
     engine: Engine,
-    httpClient client: HTTPClient
+    httpClient client: some HTTPClientProtocol
   ) async throws -> SwiftSDKProduct {
     let sdkDirPath = self.sdkDirPath(paths: generator.pathsConfiguration)
     if !generator.isIncremental {

Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift

@@ -10,7 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-import AsyncHTTPClient
 import GeneratorEngine
 import struct SystemPackage.FilePath
 
@@ -34,7 +33,7 @@ public protocol SwiftSDKRecipe: Sendable {
   var defaultArtifactID: String { get }
 
   /// The main entrypoint of the recipe to make a Swift SDK
-  func makeSwiftSDK(generator: SwiftSDKGenerator, engine: Engine, httpClient: HTTPClient) async throws -> SwiftSDKProduct
+  func makeSwiftSDK(generator: SwiftSDKGenerator, engine: Engine, httpClient: some HTTPClientProtocol) async throws -> SwiftSDKProduct
 }
 
 extension SwiftSDKRecipe {