Add support for a prebuilt swift-syntax library for macros (#8142)

Following the pattern used for the early Traits work, added a workspace
prebuilts manager to co-ordinate detecting when macros are using
swift-syntax, downloading the zip file for the resolved swift-syntax
version, and extracting it into the scratch directory. We add the
necessary information about that to the workspace state. We then morph
the Modules that use it to add build settings to find the necessary
modules and libraries.

---------

Co-authored-by: Doug Schaefer <dschaefer2@gmail.com>

Author: dschaefer2

Package.swift

@@ -655,6 +655,15 @@ let package = Package(
             name: "swift-package-registry",
             dependencies: ["Commands", "PackageRegistryCommand"]
         ),
+        .executableTarget(
+            /** Utility to produce the artifacts for prebuilts */
+            name: "swift-build-prebuilts",
+            dependencies: [
+                .product(name: "ArgumentParser", package: "swift-argument-parser"),
+                "Basics",
+                "Workspace",
+            ]
+        ),
 
         // MARK: Support for Swift macros, should eventually move to a plugin-based solution
 

Sources/Basics/Archiver/ZipArchiver.swift

@@ -13,6 +13,10 @@
 import Dispatch
 import struct TSCBasic.FileSystemError
 
+#if os(Windows)
+import WinSDK
+#endif
+
 /// An `Archiver` that handles ZIP archives using the command-line `zip` and `unzip` tools.
 public struct ZipArchiver: Archiver, Cancellable {
     public var supportedExtensions: Set<String> { ["zip"] }
@@ -23,6 +27,11 @@ public struct ZipArchiver: Archiver, Cancellable {
     /// Helper for cancelling in-flight requests
     private let cancellator: Cancellator
 
+    /// Absolute path to the Windows tar in the system folder
+    #if os(Windows)
+    private let windowsTar: String
+    #endif
+
     /// Creates a `ZipArchiver`.
     ///
     /// - Parameters:
@@ -31,6 +40,19 @@ public struct ZipArchiver: Archiver, Cancellable {
     public init(fileSystem: FileSystem, cancellator: Cancellator? = .none) {
         self.fileSystem = fileSystem
         self.cancellator = cancellator ?? Cancellator(observabilityScope: .none)
+
+        #if os(Windows)
+        var tarPath: PWSTR?
+        defer { CoTaskMemFree(tarPath) }
+        let hr = withUnsafePointer(to: FOLDERID_System) { id in
+            SHGetKnownFolderPath(id, DWORD(KF_FLAG_DEFAULT.rawValue), nil, &tarPath)
+        }
+        if hr == S_OK, let tarPath {
+            windowsTar = String(decodingCString: tarPath, as: UTF16.self) + "\\tar.exe"
+        } else {
+            windowsTar = "tar.exe"
+        }
+        #endif
     }
 
     public func extract(
@@ -48,7 +70,9 @@ public struct ZipArchiver: Archiver, Cancellable {
             }
 
             #if os(Windows)
-            let process = AsyncProcess(arguments: ["tar.exe", "xf", archivePath.pathString, "-C", destinationPath.pathString])
+            // FileManager lost the ability to detect tar.exe as executable.
+            // It's part of system32 anyway so use the absolute path.
+            let process = AsyncProcess(arguments: [windowsTar, "xf", archivePath.pathString, "-C", destinationPath.pathString])
             #else
             let process = AsyncProcess(arguments: ["unzip", archivePath.pathString, "-d", destinationPath.pathString])
             #endif
@@ -82,7 +106,7 @@ public struct ZipArchiver: Archiver, Cancellable {
         #if os(Windows)
         let process = AsyncProcess(
             // FIXME: are these the right arguments?
-            arguments: ["tar.exe", "-a", "-c", "-f", destinationPath.pathString, directory.basename],
+            arguments: [windowsTar, "-a", "-c", "-f", destinationPath.pathString, directory.basename],
             workingDirectory: directory.parentDirectory
         )
         #else
@@ -121,7 +145,7 @@ public struct ZipArchiver: Archiver, Cancellable {
             }
 
             #if os(Windows)
-            let process = AsyncProcess(arguments: ["tar.exe", "tf", path.pathString])
+            let process = AsyncProcess(arguments: [windowsTar, "tf", path.pathString])
             #else
             let process = AsyncProcess(arguments: ["unzip", "-t", path.pathString])
             #endif

Sources/Basics/Collections/IdentifiableSet.swift

@@ -122,3 +122,9 @@ extension IdentifiableSet: Hashable {
         }
     }
 }
+
+extension IdentifiableSet: ExpressibleByArrayLiteral {
+    public init(arrayLiteral elements: Element...) {
+        self.init(elements)
+    }
+}

Sources/Commands/CommandWorkspaceDelegate.swift

@@ -182,6 +182,42 @@ final class CommandWorkspaceDelegate: WorkspaceDelegate {
         self.progressHandler(step, total, "Downloading \(artifacts)")
     }
 
+    /// The workspace has started downloading a binary artifact.
+    func willDownloadPrebuilt(from url: String, fromCache: Bool) {
+        if fromCache {
+            self.outputHandler("Fetching package prebuilt \(url) from cache", false)
+        } else {
+            self.outputHandler("Downloading package prebuilt \(url)", false)
+        }
+    }
+
+    /// The workspace has finished downloading a binary artifact.
+    func didDownloadPrebuilt(
+        from url: String,
+        result: Result<(path: AbsolutePath, fromCache: Bool), Error>,
+        duration: DispatchTimeInterval
+    ) {
+        guard case .success(let fetchDetails) = result, !self.observabilityScope.errorsReported else {
+            return
+        }
+
+        if fetchDetails.fromCache {
+            self.outputHandler("Fetched \(url) from cache (\(duration.descriptionInSeconds))", false)
+        } else {
+            self.outputHandler("Downloaded \(url) (\(duration.descriptionInSeconds))", false)
+        }
+    }
+
+    /// The workspace is downloading a binary artifact.
+    func downloadingPrebuilt(from url: String, bytesDownloaded: Int64, totalBytesToDownload: Int64?) {
+
+    }
+
+    /// The workspace finished downloading all binary artifacts.
+    func didDownloadAllPrebuilts() {
+
+    }
+
     // registry signature handlers
 
     func onUnsignedRegistryPackage(registryURL: URL, package: PackageModel.PackageIdentity, version: TSCUtility.Version, completion: (Bool) -> Void) {

Sources/CoreCommands/Options.swift

@@ -186,6 +186,12 @@ public struct CachingOptions: ParsableArguments {
             self.init(rawValue: argument)
         }
     }
+
+    /// Whether to use macro prebuilts or not
+    @Flag(name: .customLong("experimental-prebuilts"),
+          inversion: .prefixedEnableDisable,
+          help: "Whether to use prebuilt swift-syntax libraries for macros")
+    public var usePrebuilts: Bool = false
 }
 
 public struct LoggingOptions: ParsableArguments {

Sources/CoreCommands/SwiftCommandState.swift

@@ -80,6 +80,7 @@ public typealias WorkspaceDelegateProvider = (
     _ progressHandler: @escaping (Int64, Int64, String?) -> Void,
     _ inputHandler: @escaping (String, (String?) -> Void) -> Void
 ) -> WorkspaceDelegate
+
 public typealias WorkspaceLoaderProvider = (_ fileSystem: FileSystem, _ observabilityScope: ObservabilityScope)
     -> WorkspaceLoader
 
@@ -470,7 +471,8 @@ public final class SwiftCommandState {
                     // TODO: should supportsAvailability be a flag as well?
                     .init(url: $0, supportsAvailability: true)
                 },
-                manifestImportRestrictions: .none
+                manifestImportRestrictions: .none,
+                usePrebuilts: options.caching.usePrebuilts
             ),
             cancellator: self.cancellator,
             initializationWarningHandler: { self.observabilityScope.emit(warning: $0) },

Sources/PackageGraph/ModulesGraph+Loading.swift

@@ -29,6 +29,7 @@ extension ModulesGraph {
         requiredDependencies: [PackageReference] = [],
         unsafeAllowedPackages: Set<PackageReference> = [],
         binaryArtifacts: [PackageIdentity: [String: BinaryArtifact]],
+        prebuilts: [PackageIdentity: [String: PrebuiltLibrary]],
         shouldCreateMultipleTestProducts: Bool = false,
         createREPLProduct: Bool = false,
         customPlatformsRegistry: PlatformRegistry? = .none,
@@ -47,6 +48,7 @@ extension ModulesGraph {
             requiredDependencies: requiredDependencies,
             unsafeAllowedPackages: unsafeAllowedPackages,
             binaryArtifacts: binaryArtifacts,
+            prebuilts: prebuilts,
             shouldCreateMultipleTestProducts: shouldCreateMultipleTestProducts,
             createREPLProduct: createREPLProduct,
             traitConfiguration: nil,
@@ -69,6 +71,7 @@ extension ModulesGraph {
         requiredDependencies: [PackageReference] = [],
         unsafeAllowedPackages: Set<PackageReference> = [],
         binaryArtifacts: [PackageIdentity: [String: BinaryArtifact]],
+        prebuilts: [PackageIdentity: [String: PrebuiltLibrary]], // Product name to library mapping
         shouldCreateMultipleTestProducts: Bool = false,
         createREPLProduct: Bool = false,
         traitConfiguration: TraitConfiguration? = nil,
@@ -212,7 +215,8 @@ extension ModulesGraph {
                     productFilter: node.productFilter,
                     path: packagePath,
                     additionalFileRules: additionalFileRules,
-                    binaryArtifacts: binaryArtifacts[node.identity] ?? [:],                
+                    binaryArtifacts: binaryArtifacts[node.identity] ?? [:],
+                    prebuilts: prebuilts,
                     shouldCreateMultipleTestProducts: shouldCreateMultipleTestProducts,
                     testEntryPointPath: testEntryPointPath,
                     createREPLProduct: manifest.packageKind.isRoot ? createREPLProduct : false,
@@ -246,6 +250,7 @@ extension ModulesGraph {
             manifestToPackage: manifestToPackage,
             rootManifests: root.manifests,
             unsafeAllowedPackages: unsafeAllowedPackages,
+            prebuilts: prebuilts,
             platformRegistry: customPlatformsRegistry ?? .default,
             platformVersionProvider: platformVersionProvider,
             fileSystem: fileSystem,
@@ -377,6 +382,7 @@ private func createResolvedPackages(
     // FIXME: This shouldn't be needed once <rdar://problem/33693433> is fixed.
     rootManifests: [PackageIdentity: Manifest],
     unsafeAllowedPackages: Set<PackageReference>,
+    prebuilts: [PackageIdentity: [String: PrebuiltLibrary]],
     platformRegistry: PlatformRegistry,
     platformVersionProvider: PlatformVersionProvider,
     fileSystem: FileSystem,
@@ -665,6 +671,11 @@ private func createResolvedPackages(
                     }
                 }
 
+                if let package = productRef.package, prebuilts[.plain(package)]?[productRef.name] != nil {
+                    // using a prebuilt instead.
+                    continue
+                }
+
                 // Find the product in this package's dependency products.
                 // Look it up by ID if module aliasing is used, otherwise by name.
                 let product = lookupByProductIDs ? productDependencyMap[productRef.identity] : productDependencyMap[productRef.name]

Sources/PackageGraph/ModulesGraph.swift

@@ -419,6 +419,7 @@ public func loadModulesGraph(
     fileSystem: FileSystem,
     manifests: [Manifest],
     binaryArtifacts: [PackageIdentity: [String: BinaryArtifact]] = [:],
+    prebuilts: [PackageIdentity: [String: PrebuiltLibrary]] = [:],
     explicitProduct: String? = .none,
     shouldCreateMultipleTestProducts: Bool = false,
     createREPLProduct: Bool = false,
@@ -451,6 +452,7 @@ public func loadModulesGraph(
             .swiftpmFileTypes,
         externalManifests: externalManifests,
         binaryArtifacts: binaryArtifacts,
+        prebuilts: prebuilts,
         shouldCreateMultipleTestProducts: shouldCreateMultipleTestProducts,
         createREPLProduct: createREPLProduct,
         customXCTestMinimumDeploymentTargets: customXCTestMinimumDeploymentTargets,

Sources/PackageLoading/PackageBuilder.swift

@@ -268,6 +268,32 @@ public struct BinaryArtifact {
     }
 }
 
+/// A structure representing a prebuilt library to be used instead of a source dependency
+public struct PrebuiltLibrary {
+    /// The package reference.
+    public let packageRef: PackageReference
+
+    /// The name of the binary target the artifact corresponds to.
+    public let libraryName: String
+
+    /// The path to the extracted prebuilt artifacts
+    public let path: AbsolutePath
+
+    /// The products in the library
+    public let products: [String]
+
+    /// The C modules that need their includes directory added to the include path
+    public let cModules: [String]
+
+    public init(packageRef: PackageReference, libraryName: String, path: AbsolutePath, products: [String], cModules: [String]) {
+        self.packageRef = packageRef
+        self.libraryName = libraryName
+        self.path = path
+        self.products = products
+        self.cModules = cModules
+    }
+}
+
 /// Helper for constructing a package following the convention system.
 ///
 /// The 'builder' here refers to the builder pattern and not any build system
@@ -295,6 +321,9 @@ public final class PackageBuilder {
     /// Information concerning the different downloaded or local (archived) binary target artifacts.
     private let binaryArtifacts: [String: BinaryArtifact]
 
+    /// Prebuilts that may referenced from this package's targets
+    private let prebuilts: [PackageIdentity: [Product.ID: PrebuiltLibrary]]
+
     /// Create multiple test products.
     ///
     /// If set to true, one test product will be created for each test target.
@@ -351,6 +380,7 @@ public final class PackageBuilder {
         path: AbsolutePath,
         additionalFileRules: [FileRuleDescription],
         binaryArtifacts: [String: BinaryArtifact],
+        prebuilts: [PackageIdentity: [String: PrebuiltLibrary]],
         shouldCreateMultipleTestProducts: Bool = false,
         testEntryPointPath: AbsolutePath? = nil,
         warnAboutImplicitExecutableTargets: Bool = true,
@@ -365,6 +395,7 @@ public final class PackageBuilder {
         self.packagePath = path
         self.additionalFileRules = additionalFileRules
         self.binaryArtifacts = binaryArtifacts
+        self.prebuilts = prebuilts
         self.shouldCreateMultipleTestProducts = shouldCreateMultipleTestProducts
         self.testEntryPointPath = testEntryPointPath
         self.createREPLProduct = createREPLProduct
@@ -1211,6 +1242,33 @@ public final class PackageBuilder {
             table.add(assignment, for: .SWIFT_ACTIVE_COMPILATION_CONDITIONS)
         }
 
+        // Add in flags for prebuilts
+        let prebuiltLibraries: [String: PrebuiltLibrary] = target.dependencies.reduce(into: .init()) {
+            guard case let .product(name: name, package: package, moduleAliases: _, condition: _) = $1,
+                  let package = package,
+                  let prebuilt = prebuilts[.plain(package)]?[name]
+            else {
+                return
+            }
+
+            $0[prebuilt.libraryName] = prebuilt
+        }
+
+        for prebuilt in prebuiltLibraries.values {
+            let lib = prebuilt.path.appending(components: ["lib", "lib\(prebuilt.libraryName).a"]).pathString
+            var ldFlagsAssignment = BuildSettings.Assignment()
+            ldFlagsAssignment.values = [lib]
+            table.add(ldFlagsAssignment, for: .OTHER_LDFLAGS)
+
+            var includeDirs: [AbsolutePath] = [prebuilt.path.appending(component: "Modules")]
+            for cModule in prebuilt.cModules {
+                includeDirs.append(prebuilt.path.appending(components: "include", cModule))
+            }
+            var includeAssignment = BuildSettings.Assignment()
+            includeAssignment.values = includeDirs.map({ "-I\($0.pathString)" })
+            table.add(includeAssignment, for: .OTHER_SWIFT_FLAGS)
+        }
+
         return table
     }
 

Sources/PackageModel/Product.swift

@@ -12,10 +12,12 @@
 
 import Basics
 
-public class Product {
+public class Product: Identifiable {
     /// The name of the product.
     public let name: String
 
+    public var id: String { name }
+
     /// Fully qualified name for this product: package ID + name of this product
     public let identity: String