Add support for generating Swift SDKs with multiple host triples (#232)

This is necessary for correctness on macOS, where the concept of multi-arch binaries exists, and all swift.org toolchains for macOS are multi-arch.

Author: jakepetroules

Sources/GeneratorCLI/GeneratorCLI.swift

@@ -141,12 +141,13 @@ extension GeneratorCLI {
     var targetSwiftPackagePath: String? = nil
 
     @Option(
+      name: .customLong("host"),
       help: """
-        The host triple of the bundle. Defaults to a triple of the machine this generator is \
-        running on if unspecified.
+        The host triples of the bundle. Defaults to a triple or triples of the machine this generator is \
+        running on if unspecified. Multiple host triples are only supported for macOS hosts.
         """
     )
-    var host: Triple? = nil
+    var hosts: [Triple] = []
 
     @Option(
       help:
@@ -174,19 +175,26 @@ extension GeneratorCLI {
       #endif
     }
 
-    func deriveHostTriple() throws -> Triple {
-      if let host {
-        return host
+    func deriveHostTriples() throws -> [Triple] {
+      if !hosts.isEmpty {
+        return hosts
       }
       let current = try SwiftSDKGenerator.getCurrentTriple(isVerbose: self.verbose)
       if let arch = hostArch {
         let target = Triple(arch: arch, vendor: current.vendor!, os: current.os!)
         appLogger.warning(
           "deprecated: Please use `--host \(target.triple)` instead of `--host-arch \(arch)`"
         )
-        return target
+        return [target]
+      }
+      // macOS toolchains are built as universal binaries
+      if current.isMacOSX {
+        return [
+          Triple("arm64-apple-macos"),
+          Triple("x86_64-apple-macos"),
+        ]
       }
-      return current
+      return [current]
     }
   }
 
@@ -230,8 +238,8 @@ extension GeneratorCLI {
     )
     var distributionVersion: String?
 
-    func deriveTargetTriple(hostTriple: Triple) -> Triple {
-      if let target = generatorOptions.target {
+    func deriveTargetTriple(hostTriples: [Triple]) -> Triple {
+      if let target = generatorOptions.target, target.os == .linux {
         return target
       }
       if let arch = generatorOptions.targetArch {
@@ -241,7 +249,13 @@ extension GeneratorCLI {
         )
         return target
       }
-      return Triple(arch: hostTriple.arch!, vendor: nil, os: .linux, environment: .gnu)
+      let arch: Triple.Arch
+      if hostTriples.count == 1, let hostTriple = hostTriples.first {
+        arch = hostTriple.arch!
+      } else {
+        arch = try! SwiftSDKGenerator.getCurrentTriple(isVerbose: false).arch!
+      }
+      return Triple(arch: arch, vendor: nil, os: .linux, environment: .gnu)
     }
 
     func run() async throws {
@@ -265,12 +279,12 @@ extension GeneratorCLI {
         name: distributionName,
         version: distributionVersion
       )
-      let hostTriple = try self.generatorOptions.deriveHostTriple()
-      let targetTriple = self.deriveTargetTriple(hostTriple: hostTriple)
+      let hostTriples = try self.generatorOptions.deriveHostTriples()
+      let targetTriple = self.deriveTargetTriple(hostTriples: hostTriples)
 
       let recipe = try LinuxRecipe(
         targetTriple: targetTriple,
-        hostTriple: hostTriple,
+        hostTriples: hostTriples,
         linuxDistribution: linuxDistribution,
         swiftVersion: generatorOptions.swiftVersion,
         swiftBranch: self.generatorOptions.swiftBranch,
@@ -331,7 +345,7 @@ extension GeneratorCLI {
     )
     var freeBSDVersion: String
 
-    func deriveTargetTriple() throws -> Triple {
+    func deriveTargetTriple(hostTriples: [Triple]) throws -> Triple {
       if let target = generatorOptions.target, target.os == .freeBSD {
         return target
       }
@@ -344,12 +358,14 @@ extension GeneratorCLI {
           """
         )
         return target
+      }
+      let arch: Triple.Arch
+      if hostTriples.count == 1, let hostTriple = hostTriples.first {
+        arch = hostTriple.arch!
       } else {
-        // If no target is given, use the architecture of the host.
-        let hostTriple = try self.generatorOptions.deriveHostTriple()
-        let hostArch = hostTriple.arch!
-        return Triple(arch: hostArch, vendor: nil, os: .freeBSD)
+        arch = try! SwiftSDKGenerator.getCurrentTriple(isVerbose: false).arch!
       }
+      return Triple(arch: arch, vendor: nil, os: .freeBSD)
     }
 
     func run() async throws {
@@ -358,8 +374,8 @@ extension GeneratorCLI {
         throw StringError("Only FreeBSD versions 14.3 or higher are supported.")
       }
 
-      let hostTriple = try self.generatorOptions.deriveHostTriple()
-      let targetTriple = try self.deriveTargetTriple()
+      let hostTriples = try self.generatorOptions.deriveHostTriples()
+      let targetTriple = try self.deriveTargetTriple(hostTriples: hostTriples)
 
       let sourceSwiftToolchain: FilePath?
       if let fromSwiftToolchain {
@@ -411,8 +427,8 @@ extension GeneratorCLI {
       }
       let recipe = try WebAssemblyRecipe(
         hostSwiftPackage: generatorOptions.hostSwiftPackagePath.map {
-          let hostTriple = try self.generatorOptions.deriveHostTriple()
-          return WebAssemblyRecipe.HostToolchainPackage(path: FilePath($0), triple: hostTriple)
+          let hostTriples = try self.generatorOptions.deriveHostTriples()
+          return WebAssemblyRecipe.HostToolchainPackage(path: FilePath($0), triples: hostTriples)
         },
         targetSwiftPackagePath: FilePath(targetSwiftPackagePath),
         wasiSysroot: FilePath(self.wasiSysroot),

Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift

@@ -18,6 +18,8 @@ import struct SystemPackage.FilePath
 extension Triple {
   fileprivate var llvmBinaryURLSuffix: String {
     switch (self.os, self.arch) {
+    case (.win32, .aarch64): return "aarch64-pc-windows-msvc"
+    case (.win32, .x86_64): return "x86_64-pc-windows-msvc"
     case (.linux, .aarch64): return "aarch64-linux-gnu"
     case (.linux, .x86_64): return "x86_64-linux-gnu-ubuntu-22.04"
     case (.macosx, .aarch64): return "arm64-apple-darwin22.0"
@@ -30,6 +32,27 @@ extension Triple {
 typealias CPUMapping = [Triple.Arch: String]
 
 struct DownloadableArtifacts: Sendable {
+  enum Error: Swift.Error, CustomStringConvertible {
+    case noHostTriples
+    case unsupportedHostTriple(Triple)
+    case unsupportedHostTriples([Triple])
+
+    var description: String {
+      switch self {
+      case .noHostTriples:
+        "no host triples"
+      case let .unsupportedHostTriple(hostTriple):
+        "unsupported host triple \(hostTriple)"
+      case let .unsupportedHostTriples(hostTriples):
+        """
+        unsupported host triples \(
+          hostTriples.map { "\($0)" }.joined(separator: ", ")
+        ) (only macOS supports multiple host triples)
+        """
+      }
+    }
+  }
+
   struct Item: Sendable, CacheKey {
     let remoteURL: URL
     var localPath: FilePath
@@ -44,15 +67,30 @@ struct DownloadableArtifacts: Sendable {
   private let paths: PathsConfiguration
 
   init(
-    hostTriple: Triple,
+    hostTriples: [Triple],
     targetTriple: Triple,
     _ versions: VersionsConfiguration,
     _ paths: PathsConfiguration
   ) throws {
     self.versions = versions
     self.paths = paths
 
-    if hostTriple.os == .linux {
+    guard let hostTriple = hostTriples.first else {
+      throw Error.noHostTriples
+    }
+
+    if hostTriples.allSatisfy({ $0.os == .macosx }) {
+      self.hostSwift = .init(
+        remoteURL: versions.swiftDownloadURL(
+          subdirectory: "xcode",
+          platform: "osx",
+          fileExtension: "pkg"
+        ),
+        localPath: paths.artifactsCachePath
+          .appending("host_swift_\(versions.swiftVersion)_apple-macos.pkg"),
+        isPrebuilt: true
+      )
+    } else if hostTriple.os == .linux && hostTriples.count == 1 {
       // Amazon Linux 2 is chosen for its best compatibility with all Swift-supported Linux hosts
       let hostArchSuffix =
         hostTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : ""
@@ -67,16 +105,11 @@ struct DownloadableArtifacts: Sendable {
         isPrebuilt: true
       )
     } else {
-      self.hostSwift = .init(
-        remoteURL: versions.swiftDownloadURL(
-          subdirectory: "xcode",
-          platform: "osx",
-          fileExtension: "pkg"
-        ),
-        localPath: paths.artifactsCachePath
-          .appending("host_swift_\(versions.swiftVersion)_\(hostTriple.triple).pkg"),
-        isPrebuilt: true
-      )
+      if hostTriples.count > 1 {
+        throw Error.unsupportedHostTriples(hostTriples)
+      } else {
+        throw Error.unsupportedHostTriple(hostTriple)
+      }
     }
 
     self.hostLLVM = .init(

Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift

@@ -30,7 +30,7 @@ package struct LinuxRecipe: SwiftSDKRecipe {
   }
 
   let mainTargetTriple: Triple
-  let mainHostTriple: Triple
+  let mainHostTriples: [Triple]
   let linuxDistribution: LinuxDistribution
   let targetSwiftSource: TargetSwiftSource
   let hostSwiftSource: HostSwiftSource
@@ -46,7 +46,7 @@ package struct LinuxRecipe: SwiftSDKRecipe {
 
   package init(
     targetTriple: Triple,
-    hostTriple: Triple,
+    hostTriples: [Triple],
     linuxDistribution: LinuxDistribution,
     swiftVersion: String,
     swiftBranch: String?,
@@ -89,7 +89,7 @@ package struct LinuxRecipe: SwiftSDKRecipe {
 
     self.init(
       mainTargetTriple: targetTriple,
-      mainHostTriple: hostTriple,
+      mainHostTriples: hostTriples,
       linuxDistribution: linuxDistribution,
       targetSwiftSource: targetSwiftSource,
       hostSwiftSource: hostSwiftSource,
@@ -100,15 +100,15 @@ package struct LinuxRecipe: SwiftSDKRecipe {
 
   package init(
     mainTargetTriple: Triple,
-    mainHostTriple: Triple,
+    mainHostTriples: [Triple],
     linuxDistribution: LinuxDistribution,
     targetSwiftSource: TargetSwiftSource,
     hostSwiftSource: HostSwiftSource,
     versionsConfiguration: VersionsConfiguration,
     logger: Logger
   ) {
     self.mainTargetTriple = mainTargetTriple
-    self.mainHostTriple = mainHostTriple
+    self.mainHostTriples = mainHostTriples
     self.linuxDistribution = linuxDistribution
     self.targetSwiftSource = targetSwiftSource
     self.hostSwiftSource = hostSwiftSource
@@ -180,7 +180,7 @@ package struct LinuxRecipe: SwiftSDKRecipe {
   func itemsToDownload(from artifacts: DownloadableArtifacts) -> [DownloadableArtifacts.Item] {
     var items: [DownloadableArtifacts.Item] = []
     if self.hostSwiftSource != .preinstalled
-      && self.mainHostTriple.os != .linux
+      && !self.mainHostTriples.contains(where: { $0.os == .linux })
       && !self.versionsConfiguration.swiftVersion.hasPrefix("6.")
     {
       items.append(artifacts.hostLLVM)
@@ -218,7 +218,7 @@ package struct LinuxRecipe: SwiftSDKRecipe {
       return nil
     }
 
-    return [self.mainHostTriple]
+    return self.mainHostTriples
   }
 
   package func makeSwiftSDK(
@@ -240,7 +240,7 @@ package struct LinuxRecipe: SwiftSDKRecipe {
     try await generator.createDirectoryIfNeeded(at: sdkDirPath)
 
     var downloadableArtifacts = try DownloadableArtifacts(
-      hostTriple: mainHostTriple,
+      hostTriples: mainHostTriples,
       targetTriple: generator.targetTriple,
       self.versionsConfiguration,
       generator.pathsConfiguration
@@ -335,7 +335,7 @@ package struct LinuxRecipe: SwiftSDKRecipe {
     }
 
     if self.hostSwiftSource != .preinstalled {
-      if self.mainHostTriple.os != .linux
+      if !self.mainHostTriples.contains(where: { $0.os == .linux })
         && !self.versionsConfiguration.swiftVersion.hasPrefix("6.")
       {
         try await generator.prepareLLDLinker(engine, llvmArtifact: downloadableArtifacts.hostLLVM)

Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift

@@ -24,11 +24,11 @@ package struct WebAssemblyRecipe: SwiftSDKRecipe {
 
   package struct HostToolchainPackage: Sendable {
     let path: FilePath
-    let triple: Triple
+    let triples: [Triple]
 
-    package init(path: FilePath, triple: Triple) {
+    package init(path: FilePath, triples: [Triple]) {
       self.path = path
-      self.triple = triple
+      self.triples = triples
     }
   }
 
@@ -145,7 +145,7 @@ package struct WebAssemblyRecipe: SwiftSDKRecipe {
     logger.info("Copying Swift binaries for the host triple...")
     var hostTriples: [Triple]? = nil
     if let hostSwiftPackage {
-      hostTriples = [hostSwiftPackage.triple]
+      hostTriples = hostSwiftPackage.triples
       try await generator.rsync(
         from: hostSwiftPackage.path.appending("usr"),
         to: pathsConfiguration.toolchainDirPath

Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift

@@ -49,7 +49,7 @@ final class ArchitectureMappingTests: XCTestCase {
   ) async throws {
     let recipe = try LinuxRecipe(
       targetTriple: targetTriple,
-      hostTriple: hostTriple,
+      hostTriples: [hostTriple],
       linuxDistribution: .ubuntu(.jammy),
       swiftVersion: "5.8-RELEASE",
       swiftBranch: nil,
@@ -79,7 +79,7 @@ final class ArchitectureMappingTests: XCTestCase {
 
     // Verify download URLs
     let artifacts = try await DownloadableArtifacts(
-      hostTriple: hostTriple,
+      hostTriples: [hostTriple],
       targetTriple: sdk.targetTriple,
       recipe.versionsConfiguration,
       sdk.pathsConfiguration