Allow the "generic Unix" platforms as cross-compile targets (#749)

This allows Linux, FreeBSD, and OpenBSD to be used with Swift SDKs for cross compilation, not only as part of the toolchain on the same host as the target.

Author: jakepetroules

Sources/SWBCore/SwiftSDK.swift

@@ -85,11 +85,11 @@ public struct SwiftSDK: Sendable {
     }
 
     /// Find Swift SDKs installed by SwiftPM.
-    public static func findSDKs(targetTriples: [String], fs: any FSProxy, hostOperatingSystem: OperatingSystem) throws -> [SwiftSDK] {
+    public static func findSDKs(targetTriples: [String]?, fs: any FSProxy, hostOperatingSystem: OperatingSystem) throws -> [SwiftSDK] {
         return try findSDKs(swiftSDKsDirectory: defaultSwiftSDKsDirectory(hostOperatingSystem: hostOperatingSystem), targetTriples: targetTriples, fs: fs)
     }
 
-    private static func findSDKs(swiftSDKsDirectory: Path, targetTriples: [String], fs: any FSProxy) throws -> [SwiftSDK] {
+    private static func findSDKs(swiftSDKsDirectory: Path, targetTriples: [String]?, fs: any FSProxy) throws -> [SwiftSDK] {
         var sdks: [SwiftSDK] = []
         // Find .artifactbundle in the SDK directory (e.g. ~/Library/org.swift.swiftpm/swift-sdks)
         for artifactBundle in try fs.listdir(swiftSDKsDirectory) {
@@ -118,7 +118,7 @@ public struct SwiftSDK: Sendable {
     }
 
     /// Find Swift SDKs in an artifact bundle supporting one of the given targets.
-    private static func findSDKs(artifactBundle: Path, targetTriples: [String], fs: any FSProxy) throws -> [SwiftSDK] {
+    private static func findSDKs(artifactBundle: Path, targetTriples: [String]?, fs: any FSProxy) throws -> [SwiftSDK] {
         // Load info.json from the artifact bundle
         let infoPath = artifactBundle.join("info.json")
         guard try fs.isFile(infoPath) else { return [] }
@@ -145,7 +145,9 @@ public struct SwiftSDK: Sendable {
 
                 guard let sdk = try SwiftSDK(identifier: identifier, version: artifact.version, path: sdkPath, fs: fs) else { continue }
                 // Filter out SDKs that don't support any of the target triples.
-                guard targetTriples.contains(where: { sdk.targetTriples[$0] != nil }) else { continue }
+                if let targetTriples {
+                    guard targetTriples.contains(where: { sdk.targetTriples[$0] != nil }) else { continue }
+                }
                 sdks.append(sdk)
             }
         }

Sources/SWBGenericUnixPlatform/Plugin.swift

@@ -49,12 +49,6 @@ struct SwiftTargetInfo: Decodable {
     let target: TargetInfo
 }
 
-extension SwiftTargetInfo.TargetInfo {
-    var tripleVersion: String? {
-        triple != unversionedTriple && triple.system.hasPrefix(unversionedTriple.system) ? String(triple.system.dropFirst(unversionedTriple.system.count)).nilIfEmpty : nil
-    }
-}
-
 struct GenericUnixDeveloperDirectoryExtension: DeveloperDirectoryExtension {
     func fallbackDeveloperDirectory(hostOperatingSystem: OperatingSystem) async throws -> Core.DeveloperPath? {
         if hostOperatingSystem == .windows || hostOperatingSystem == .macOS {
@@ -82,13 +76,12 @@ struct GenericUnixPlatformSpecsExtension: SpecificationsExtension {
 
 struct GenericUnixPlatformInfoExtension: PlatformInfoExtension {
     func additionalPlatforms(context: any PlatformInfoExtensionAdditionalPlatformsContext) throws -> [(path: Path, data: [String: PropertyListItem])] {
-        let operatingSystem = context.hostOperatingSystem
-        guard operatingSystem.createFallbackSystemToolchain else {
-            return []
-        }
-
-        return try [
-            (.root, [
+        return try OperatingSystem.createFallbackSystemToolchains.compactMap { operatingSystem in
+            // Only create platforms if the host OS allows a fallback toolchain, or we're cross compiling.
+            guard operatingSystem.createFallbackSystemToolchain || operatingSystem != context.hostOperatingSystem else {
+                return nil
+            }
+            return try (.root, [
                 "Type": .plString("Platform"),
                 "Name": .plString(operatingSystem.xcodePlatformName),
                 "Identifier": .plString(operatingSystem.xcodePlatformName),
@@ -97,78 +90,156 @@ struct GenericUnixPlatformInfoExtension: PlatformInfoExtension {
                 "FamilyIdentifier": .plString(operatingSystem.xcodePlatformName),
                 "IsDeploymentPlatform": .plString("YES"),
             ])
-        ]
+        }
     }
 }
 
 struct GenericUnixSDKRegistryExtension: SDKRegistryExtension {
     let plugin: GenericUnixPlugin
 
     func additionalSDKs(context: any SDKRegistryExtensionAdditionalSDKsContext) async throws -> [(path: Path, platform: SWBCore.Platform?, data: [String: PropertyListItem])] {
-        let operatingSystem = context.hostOperatingSystem
-        guard operatingSystem.createFallbackSystemToolchain, let platform = try context.platformRegistry.lookup(name: operatingSystem.xcodePlatformName), let swift = plugin.swiftExecutablePath(fs: context.fs) else {
-            return []
-        }
+        return try await OperatingSystem.createFallbackSystemToolchains.asyncMap { operatingSystem in
+            // Only create SDKs if the host OS allows a fallback toolchain, or we're cross compiling.
+            guard operatingSystem.createFallbackSystemToolchain || operatingSystem != context.hostOperatingSystem else {
+                return nil
+            }
 
-        let defaultProperties: [String: PropertyListItem]
-        switch operatingSystem {
-        case .linux, .freebsd:
-            defaultProperties = [
-                // Workaround to avoid `-dependency_info`.
-                "LD_DEPENDENCY_INFO_FILE": .plString(""),
-
-                "GENERATE_TEXT_BASED_STUBS": "NO",
-                "GENERATE_INTERMEDIATE_TEXT_BASED_STUBS": "NO",
-
-                "CHOWN": "/usr/bin/chown",
-                "AR": "llvm-ar",
-            ]
-        default:
-            defaultProperties = [:]
-        }
+            // Don't create any SDKs for the platform if the platform itself isn't registered.
+            guard let platform = try context.platformRegistry.lookup(name: operatingSystem.xcodePlatformName) else {
+                return nil
+            }
 
-        let tripleEnvironment: String
-        switch operatingSystem {
-        case .linux:
-            tripleEnvironment = "gnu"
-        default:
-            tripleEnvironment = ""
-        }
+            var defaultProperties: [String: PropertyListItem]
+            switch operatingSystem {
+            case .linux, .freebsd:
+                defaultProperties = [
+                    // Workaround to avoid `-dependency_info`.
+                    "LD_DEPENDENCY_INFO_FILE": .plString(""),
 
-        let swiftTargetInfo = try await plugin.swiftTargetInfo(swiftExecutablePath: swift)
+                    "GENERATE_TEXT_BASED_STUBS": "NO",
+                    "GENERATE_INTERMEDIATE_TEXT_BASED_STUBS": "NO",
 
-        let deploymentTargetSettings: [String: PropertyListItem]
-        if operatingSystem == .freebsd {
-            guard let tripleVersion = swiftTargetInfo.target.tripleVersion else {
-                throw StubError.error("Unknown FreeBSD triple version")
+                    "CHOWN": "/usr/bin/chown",
+                    "AR": "llvm-ar",
+                ]
+            default:
+                defaultProperties = [:]
             }
-            deploymentTargetSettings = [
-                "DeploymentTargetSettingName": .plString("FREEBSD_DEPLOYMENT_TARGET"),
-                "DefaultDeploymentTarget": .plString(tripleVersion),
-                "MinimumDeploymentTarget": .plString(tripleVersion),
-                "MaximumDeploymentTarget": .plString(tripleVersion),
-            ]
-        } else {
-            deploymentTargetSettings = [:]
-        }
 
-        return try [(.root, platform, [
-            "Type": .plString("SDK"),
-            "Version": .plString(Version(ProcessInfo.processInfo.operatingSystemVersion).zeroTrimmed.description),
-            "CanonicalName": .plString(operatingSystem.xcodePlatformName),
-            "IsBaseSDK": .plBool(true),
-            "DefaultProperties": .plDict([
-                "PLATFORM_NAME": .plString(operatingSystem.xcodePlatformName),
-            ].merging(defaultProperties, uniquingKeysWith: { _, new in new })),
-            "SupportedTargets": .plDict([
-                operatingSystem.xcodePlatformName: .plDict([
-                    "Archs": .plArray([.plString(Architecture.hostStringValue ?? "unknown")]),
-                    "LLVMTargetTripleEnvironment": .plString(tripleEnvironment),
-                    "LLVMTargetTripleSys": .plString(operatingSystem.xcodePlatformName),
-                    "LLVMTargetTripleVendor": .plString("unknown"),
-                ].merging(deploymentTargetSettings, uniquingKeysWith: { _, new in new }))
-            ]),
-        ])]
+            if operatingSystem == .freebsd || operatingSystem != context.hostOperatingSystem {
+                // FreeBSD is always LLVM-based, and if we're cross-compiling, use lld
+                defaultProperties["ALTERNATE_LINKER"] = "lld"
+            }
+
+            let tripleEnvironment: String
+            switch operatingSystem {
+            case .linux:
+                tripleEnvironment = "gnu"
+            default:
+                tripleEnvironment = ""
+            }
+
+            let swiftSDK: SwiftSDK?
+            let sysroot: Path
+            let architectures: [String]
+            let tripleVersion: String?
+            let customProperties: [String: PropertyListItem]
+            if operatingSystem == context.hostOperatingSystem {
+                swiftSDK = nil
+                sysroot = .root
+                architectures = [Architecture.hostStringValue ?? "unknown"]
+                tripleVersion = nil
+                customProperties = [:]
+            } else {
+                do {
+                    let swiftSDKs = try SwiftSDK.findSDKs(
+                        targetTriples: nil,
+                        fs: context.fs,
+                        hostOperatingSystem: context.hostOperatingSystem
+                    ).filter { sdk in
+                        try sdk.targetTriples.keys.map {
+                            try LLVMTriple($0)
+                        }.contains {
+                            switch operatingSystem {
+                            case .linux:
+                                $0.system == "linux" && $0.environment?.hasPrefix("gnu") == true
+                            case .freebsd:
+                                $0.system == "freebsd"
+                            case .openbsd:
+                                $0.system == "openbsd"
+                            default:
+                                throw StubError.error("Unhandled operating system: \(operatingSystem)")
+                            }
+                        }
+                    }
+                    // FIXME: Do something better than just skipping the platform if more than one SDK matches
+                    swiftSDK = swiftSDKs.only
+                    guard let swiftSDK else {
+                        return nil
+                    }
+                    sysroot = swiftSDK.path
+                    architectures = try swiftSDK.targetTriples.keys.map { try LLVMTriple($0).arch }.sorted()
+                    tripleVersion = try Set(swiftSDK.targetTriples.keys.compactMap { try LLVMTriple($0).systemVersion }).only?.description
+                    customProperties = try Dictionary(uniqueKeysWithValues: swiftSDK.targetTriples.map { targetTriple in
+                        try ("__SYSROOT_\(LLVMTriple(targetTriple.key).arch)", .plString(swiftSDK.path.join(targetTriple.value.sdkRootPath).str))
+                    }).merging([
+                        "SYSROOT": "$(__SYSROOT_$(CURRENT_ARCH))",
+
+                        // ld.lld: error: -r and --export-dynamic (-rdynamic) may not be used together
+                        "LD_EXPORT_GLOBAL_SYMBOLS": "YES",
+                    ], uniquingKeysWith: { _, new in new })
+                } catch {
+                    // FIXME: Handle errors?
+                    return nil
+                }
+            }
+
+            let deploymentTargetSettings: [String: PropertyListItem]
+            if operatingSystem == .freebsd {
+                let realTripleVersion: String
+                if context.hostOperatingSystem == operatingSystem {
+                    guard let swift = plugin.swiftExecutablePath(fs: context.fs) else {
+                        throw StubError.error("Cannot locate swift executable path for determining the FreeBSD triple version")
+                    }
+                    let swiftTargetInfo = try await plugin.swiftTargetInfo(swiftExecutablePath: swift)
+                    guard let foundTripleVersion = try swiftTargetInfo.target.triple.version?.description else {
+                        throw StubError.error("Unknown FreeBSD triple version")
+                    }
+                    realTripleVersion = foundTripleVersion
+                } else if let tripleVersion {
+                    realTripleVersion = tripleVersion
+                } else {
+                    return nil // couldn't compute triple version for FreeBSD
+                }
+                deploymentTargetSettings = [
+                    "DeploymentTargetSettingName": .plString("FREEBSD_DEPLOYMENT_TARGET"),
+                    "DefaultDeploymentTarget": .plString(realTripleVersion),
+                    "MinimumDeploymentTarget": .plString(realTripleVersion),
+                    "MaximumDeploymentTarget": .plString(realTripleVersion),
+                ]
+            } else {
+                deploymentTargetSettings = [:]
+            }
+
+            return try (sysroot, platform, [
+                "Type": .plString("SDK"),
+                "Version": .plString(Version(ProcessInfo.processInfo.operatingSystemVersion).zeroTrimmed.description),
+                "CanonicalName": .plString(operatingSystem.xcodePlatformName),
+                "IsBaseSDK": .plBool(true),
+                "DefaultProperties": .plDict([
+                    "PLATFORM_NAME": .plString(operatingSystem.xcodePlatformName),
+                ].merging(defaultProperties, uniquingKeysWith: { _, new in new })),
+                "CustomProperties": .plDict(customProperties),
+                "SupportedTargets": .plDict([
+                    operatingSystem.xcodePlatformName: .plDict([
+                        "Archs": .plArray(architectures.map { .plString($0) }),
+                        "LLVMTargetTripleEnvironment": .plString(tripleEnvironment),
+                        "LLVMTargetTripleSys": .plString(operatingSystem.xcodePlatformName),
+                        "LLVMTargetTripleVendor": .plString("unknown"),
+                    ].merging(deploymentTargetSettings, uniquingKeysWith: { _, new in new }))
+                ]),
+            ])
+        }.compactMap { $0 }
     }
 }
 
@@ -218,8 +289,12 @@ struct GenericUnixToolchainRegistryExtension: ToolchainRegistryExtension {
 }
 
 extension OperatingSystem {
+    static var createFallbackSystemToolchains: [OperatingSystem] {
+        [.linux, .freebsd, .openbsd]
+    }
+
     /// Whether the Core is allowed to create a fallback toolchain, SDK, and platform for this operating system in cases where no others have been provided.
-    var createFallbackSystemToolchain: Bool {
-        return self == .linux || self == .freebsd || self == .openbsd
+    fileprivate var createFallbackSystemToolchain: Bool {
+        return Self.createFallbackSystemToolchains.contains(self)
     }
 }

Sources/SWBGenericUnixPlatform/Specs/UnixCompile.xcspec

@@ -38,6 +38,12 @@
             {
                 Name = SDKROOT;
                 Type = Path;
+                CommandLineArgs = ();
+            },
+            {
+                Name = SYSROOT;
+                DefaultValue = "$(SDKROOT)";
+                Type = Path;
                 CommandLineFlag = "--sysroot";
             },
             {
@@ -91,6 +97,12 @@
             {
                 Name = SDKROOT;
                 Type = Path;
+                CommandLineArgs = ();
+            },
+            {
+                Name = SYSROOT;
+                DefaultValue = "$(SDKROOT)";
+                Type = Path;
                 CommandLineFlag = "--sysroot";
             },
         );
@@ -106,6 +118,12 @@
             {
                 Name = SDKROOT;
                 Type = Path;
+                CommandLineArgs = ();
+            },
+            {
+                Name = SYSROOT;
+                DefaultValue = "$(SDKROOT)";
+                Type = Path;
                 CommandLineFlag = "--sysroot";
             },
         );

Sources/SWBGenericUnixPlatform/Specs/UnixLd.xcspec

@@ -53,15 +53,15 @@
             {
                 Name = CLANG_SDKROOT_LINKER_INPUT;
                 Type = Path;
-                DefaultValue = "$(SDKROOT)";
+                DefaultValue = "$(SYSROOT:default=$(SDKROOT))";
                 Condition = "$(LINKER_DRIVER) == clang";
                 CommandLineFlag = "--sysroot";
                 IsInputDependency = Yes;
             },
             {
                 Name = SWIFTC_SDKROOT_LINKER_INPUT;
                 Type = Path;
-                DefaultValue = "$(SDKROOT)";
+                DefaultValue = "$(SYSROOT:default=$(SDKROOT))";
                 Condition = "$(LINKER_DRIVER) == swiftc";
                 CommandLineFlag = "-sysroot";
                 IsInputDependency = Yes;