Allow command line tool specs to be extended by BuildSettingsExtension specs and an additional task producer extension point

This can allow more decoupling of platform specific concepts and the like from the core build specs.

Author: jakepetroules

Sources/SWBApplePlatform/AssetCatalogCompiler.swift

@@ -239,7 +239,12 @@ public final class ActoolCompilerSpec : GenericCompilerSpec, SpecIdentifierType,
         // which it currently receives a listing of via the assetcatalog_dependencies file produced by actool.
         let carFiles = [cbc.resourcesDir?.join("Assets.car")].compactMap { $0 }.map(delegate.createNode)
 
-        let outputs = evaluatedOutputsResult + (additionalEvaluatedOutputsResult.outputs ).map(delegate.createNode)
+        let outputs = evaluatedOutputsResult + (additionalEvaluatedOutputsResult.outputs).map { output in
+            if let fileTypeIdentifier = output.fileType, let fileType = cbc.producer.lookupFileType(identifier: fileTypeIdentifier) {
+                delegate.declareOutput(FileToBuild(absolutePath: output.path, fileType: fileType))
+            }
+            return delegate.createNode(output.path)
+        }
         guard !outputs.isEmpty else { preconditionFailure("ActoolCompilerSpec.constructTasks() invoked with no outputs defined") }
 
         let assetSymbolInputs = cbc.inputs

Sources/SWBApplePlatform/InterfaceBuilderCompiler.swift

@@ -86,7 +86,12 @@ public class IbtoolCompilerSpec : GenericCompilerSpec, IbtoolCompilerSupport, @u
 
         // Add the additional outputs defined by the spec.  These are not declared as outputs but should be processed by the tool separately.
         let additionalEvaluatedOutputsResult = await additionalEvaluatedOutputs(cbc, delegate)
-        outputs += additionalEvaluatedOutputsResult.outputs.map(delegate.createNode)
+        outputs += additionalEvaluatedOutputsResult.outputs.map { output in
+            if let fileTypeIdentifier = output.fileType, let fileType = cbc.producer.lookupFileType(identifier: fileTypeIdentifier) {
+                delegate.declareOutput(FileToBuild(absolutePath: output.path, fileType: fileType))
+            }
+            return delegate.createNode(output.path)
+        }
 
         if let infoPlistContent = additionalEvaluatedOutputsResult.generatedInfoPlistContent {
             delegate.declareGeneratedInfoPlistContent(infoPlistContent)

Sources/SWBApplePlatform/Plugin.swift

@@ -37,7 +37,6 @@ struct AppleDeveloperDirectoryExtension: DeveloperDirectoryExtension {
 }
 
 struct TaskProducersExtension: TaskProducerExtension {
-
     func createPreSetupTaskProducers(_ context: TaskProducerContext) -> [any TaskProducer] {
         [DevelopmentAssetsTaskProducer(context)]
     }
@@ -61,6 +60,9 @@ struct TaskProducersExtension: TaskProducerExtension {
     var globalTaskProducers: [any GlobalTaskProducerFactory] {
         [StubBinaryTaskProducerFactory()]
     }
+
+    func generateAdditionalTasks(_ tasks: inout [any SWBCore.PlannedTask], _ producer: any SWBTaskConstruction.TaskProducer) {
+    }
 }
 
 struct ExtensionPointExtractorTaskProducerFactory: TaskProducerFactory {

Sources/SWBCore/Core.swift

@@ -590,6 +590,7 @@ public final class Core: Sendable {
 
         let specs: [SpecDump] = specRegistry.domains.flatMap { domain -> [SpecDump] in
             let allSpecs = specRegistry.findSpecs(BuildSettingsSpec.self, domain: domain, includeInherited: false)
+                + specRegistry.findSpecs(BuildSettingsExtensionSpec.self, domain: domain, includeInherited: false)
                 + specRegistry.findSpecs(BuildSystemSpec.self, domain: domain, includeInherited: false)
                 + specRegistry.findSpecs(CommandLineToolSpec.self, domain: domain, includeInherited: false)
             return allSpecs.map { spec in

Sources/SWBCore/Settings/Settings.swift

@@ -83,7 +83,7 @@ fileprivate struct PreOverridesSettings {
             let ignoredMacros: [MacroDeclaration] = [BuiltinMacros.OutputFormat, BuiltinMacros.OutputPath]
 
             for spec in specs {
-                for option in spec.flattenedBuildOptions.values {
+                for option in core.specRegistry.effectiveFlattenedBuildOptions(spec).values {
                     guard !ignoredMacros.contains(option.macro) else { continue }
 
                     if let value = option.defaultValue {
@@ -236,11 +236,11 @@ fileprivate struct PreOverridesSettings {
             // Add the defaults from all the registered tools in the given domain.
             //
             // FIXME: This is somewhat wasteful, as we end up duplicating values for super specifications. However, that lets us keep the condition set required to enable a particular compiler very simple.
-            let unionedDefaults = unionedToolDefaults(domain: domain)
+            let unionedDefaults = unionedToolDefaults(domain: domain).table
             let customizedDefaults = MacroValueAssignmentTable(namespace: core.specRegistry.internalMacroNamespace)
             for spec in core.specRegistry.findSpecs(CompilerSpec.self, domain: domain) {
                 // Add all the necessary defaults.
-                for option in spec.flattenedBuildOptions.values {
+                for option in core.specRegistry.effectiveFlattenedBuildOptions(spec).values {
                     if let defaultValue = option.defaultValue {
                         // Only push the default value if it diverges from the existing default.
                         //

Sources/SWBCore/SpecImplementations/CommandLineToolSpec.swift

@@ -776,7 +776,7 @@ open class CommandLineToolSpec : PropertyDomainSpec, SpecType, TaskTypeDescripti
         let scope = cbc.scope
         let inputFileType = cbc.inputs.first?.fileType
         let lookup = { self.lookup($0, cbc, delegate) }
-        for buildOption in self.flattenedOrderedBuildOptions {
+        for buildOption in cbc.producer.effectiveFlattenedOrderedBuildOptions(self) {
             guard let dependencyFormat = buildOption.dependencyFormat else {
                 continue
             }
@@ -910,7 +910,12 @@ open class CommandLineToolSpec : PropertyDomainSpec, SpecType, TaskTypeDescripti
 
         // Add the additional outputs defined by the spec.  These are not declared as outputs but should be processed by the tool separately.
         let additionalEvaluatedOutputsResult = await additionalEvaluatedOutputs(cbc, delegate)
-        outputs.append(contentsOf: additionalEvaluatedOutputsResult.outputs.map({ delegate.createNode($0) }))
+        outputs.append(contentsOf: additionalEvaluatedOutputsResult.outputs.map { output in
+            if let fileTypeIdentifier = output.fileType, let fileType = cbc.producer.lookupFileType(identifier: fileTypeIdentifier) {
+                delegate.declareOutput(FileToBuild(absolutePath: output.path, fileType: fileType))
+            }
+            return delegate.createNode(output.path)
+        })
 
         if let infoPlistContent = additionalEvaluatedOutputsResult.generatedInfoPlistContent {
             delegate.declareGeneratedInfoPlistContent(infoPlistContent)
@@ -972,7 +977,7 @@ open class CommandLineToolSpec : PropertyDomainSpec, SpecType, TaskTypeDescripti
     }
 
     public struct AdditionalEvaluatedOutputsResult {
-        public var outputs = [Path]()
+        public var outputs = [(path: Path, fileType: String?)]()
         public var generatedInfoPlistContent: Path? = nil
     }
 
@@ -988,20 +993,25 @@ open class CommandLineToolSpec : PropertyDomainSpec, SpecType, TaskTypeDescripti
 
             // FIXME: In Xcode, this is also marked as an "auxiliary output", which we use in conjunction with the "MightNotEmitAllOutput" flag to determine whether or not the tool needs to rerun if the output is missing.
 
-            result.outputs.append(output)
+            result.outputs.append((output, nil))
         }
 
         let producer = cbc.producer
         let scope = cbc.scope
         let inputFileType = cbc.inputs.first?.fileType
         let lookup = { self.lookup($0, cbc, delegate) }
         let optionContext = await discoveredCommandLineToolSpecInfo(producer, scope, delegate)
-        result.outputs.append(contentsOf: self.flattenedOrderedBuildOptions.flatMap { buildOption -> [Path] in
+        result.outputs.append(contentsOf: cbc.producer.effectiveFlattenedOrderedBuildOptions(self, filter: .all).flatMap { buildOption -> [(Path, String?)] in
             // Check if the effective arguments for this build option were non-empty as a proxy for whether it got filtered out by architecture mismatch, etc.
             guard let outputDependencies = buildOption.outputDependencies, !buildOption.getArgumentsForCommand(producer, scope: scope, inputFileType: inputFileType, optionContext: optionContext, lookup: lookup).isEmpty else {
                 return []
             }
-            return outputDependencies.compactMap { Path(scope.evaluate($0, lookup: lookup)).nilIfEmpty?.normalize() }
+            return outputDependencies.compactMap { outputDependency in
+                guard let path = Path(scope.evaluate(outputDependency.path, lookup: lookup)).nilIfEmpty else {
+                    return nil
+                }
+                return (path.normalize(), outputDependency.fileType.map { scope.evaluate($0, lookup: lookup).nilIfEmpty } ?? nil)
+            }
         })
 
         return result
@@ -1225,7 +1235,7 @@ open class CommandLineToolSpec : PropertyDomainSpec, SpecType, TaskTypeDescripti
                 return cbc.scope.evaluate(value, lookup: lookup).map { .literal(ByteString(encodingAsUTF8: $0)) }
 
             case .options:
-                return self.commandLineFromOptions(cbc, delegate, optionContext: optionContext, lookup: lookup)
+                return self.commandLineFromOptions(cbc, delegate, optionContext: optionContext, buildOptionsFilter: .all, lookup: lookup)
 
             case .output:
                 // We always resolve the Output via a recursive macro evaluation. See constructTasks() for more information.
@@ -1254,22 +1264,22 @@ open class CommandLineToolSpec : PropertyDomainSpec, SpecType, TaskTypeDescripti
     /// Creates and returns the command line arguments generated by the options of the specification.
     ///
     /// - parameter lookup: An optional closure which functionally defined overriding values during build setting evaluation.
-    public func commandLineFromOptions(_ producer: any CommandProducer, scope: MacroEvaluationScope, inputFileType: FileTypeSpec?, optionContext: (any BuildOptionGenerationContext)?, lookup: ((MacroDeclaration) -> MacroExpression?)? = nil) -> [CommandLineArgument] {
-        return self.flattenedOrderedBuildOptions.flatMap { $0.getArgumentsForCommand(producer, scope: scope, inputFileType: inputFileType, optionContext: optionContext, lookup: lookup) }
+    public func commandLineFromOptions(_ producer: any CommandProducer, scope: MacroEvaluationScope, inputFileType: FileTypeSpec?, optionContext: (any BuildOptionGenerationContext)?, buildOptionsFilter: BuildOptionsFilter, lookup: ((MacroDeclaration) -> MacroExpression?)? = nil) -> [CommandLineArgument] {
+        return producer.effectiveFlattenedOrderedBuildOptions(self, filter: buildOptionsFilter).flatMap { $0.getArgumentsForCommand(producer, scope: scope, inputFileType: inputFileType, optionContext: optionContext, lookup: lookup) }
     }
 
     /// Creates and returns the command line arguments generated by the options of the specification.
     ///
     /// - parameter lookup: An optional closure which functionally defined overriding values during build setting evaluation.
-    public func commandLineFromOptions(_ cbc: CommandBuildContext, _ delegate: any DiagnosticProducingDelegate, optionContext: (any BuildOptionGenerationContext)?, lookup: ((MacroDeclaration) -> MacroExpression?)? = nil) -> [CommandLineArgument] {
-        return commandLineFromOptions(cbc.producer, scope: cbc.scope, inputFileType: cbc.inputs.first?.fileType, optionContext: optionContext, lookup: { self.lookup($0, cbc, delegate, lookup) })
+    public func commandLineFromOptions(_ cbc: CommandBuildContext, _ delegate: any DiagnosticProducingDelegate, optionContext: (any BuildOptionGenerationContext)?, buildOptionsFilter: BuildOptionsFilter = .all, lookup: ((MacroDeclaration) -> MacroExpression?)? = nil) -> [CommandLineArgument] {
+        return commandLineFromOptions(cbc.producer, scope: cbc.scope, inputFileType: cbc.inputs.first?.fileType, optionContext: optionContext, buildOptionsFilter: buildOptionsFilter, lookup: { self.lookup($0, cbc, delegate, lookup) })
     }
 
     /// Creates and returns the command line arguments generated by the specification's build setting corresponding to the given macro declaration.
     ///
     /// - parameter lookup: An optional closure which functionally defined overriding values during build setting evaluation.
     func commandLineFromMacroDeclaration(_ producer: any CommandProducer, optionContext: (any BuildOptionGenerationContext)?, scope: MacroEvaluationScope, macro: MacroDeclaration, inputFileType: FileTypeSpec?, lookup: ((MacroDeclaration) -> MacroExpression?)? = nil) -> [CommandLineArgument] {
-        return buildOptions.first { $0.name == macro.name }?.getArgumentsForCommand(producer, scope: scope, inputFileType: inputFileType, optionContext: optionContext, lookup: lookup) ?? []
+        return producer.effectiveBuildOptions(self).first { $0.name == macro.name }?.getArgumentsForCommand(producer, scope: scope, inputFileType: inputFileType, optionContext: optionContext, lookup: lookup) ?? []
     }
 
     /// Creates and returns the command line arguments generated by the specification's build setting corresponding to the given macro declaration.
@@ -1285,7 +1295,7 @@ open class CommandLineToolSpec : PropertyDomainSpec, SpecType, TaskTypeDescripti
         let scope = cbc.scope
         let inputFileType = cbc.inputs.first?.fileType
         let lookup = { self.lookup($0, cbc, delegate, lookup) }
-        return self.flattenedOrderedBuildOptions.flatMap { buildOption -> [Path] in
+        return cbc.producer.effectiveFlattenedOrderedBuildOptions(self).flatMap { buildOption -> [Path] in
             // Check if the effective arguments for this build option were non-empty as a proxy for whether it got filtered out by architecture mismatch, etc.
             guard let inputInclusions = buildOption.inputInclusions, !buildOption.getArgumentsForCommand(producer, scope: scope, inputFileType: inputFileType, optionContext: optionContext, lookup: lookup).isEmpty else {
                 return []
@@ -1297,7 +1307,7 @@ open class CommandLineToolSpec : PropertyDomainSpec, SpecType, TaskTypeDescripti
     /// Compute the list of additional linker arguments to use when this tool is used for building with the given scope.
     public func computeAdditionalLinkerArgs(_ producer: any CommandProducer, scope: MacroEvaluationScope, inputFileTypes: [FileTypeSpec], optionContext: (any BuildOptionGenerationContext)?, delegate: any TaskGenerationDelegate) async -> (args: [[String]], inputPaths: [Path]) {
         // FIXME: Optimize the list to search here.
-        return (args: self.flattenedOrderedBuildOptions.map { $0.getAdditionalLinkerArgs(producer, scope: scope, inputFileTypes: inputFileTypes) }, inputPaths: [])
+        return (args: producer.effectiveFlattenedOrderedBuildOptions(self).map { $0.getAdditionalLinkerArgs(producer, scope: scope, inputFileTypes: inputFileTypes) }, inputPaths: [])
     }
 
     // Creates and returns the environment from the specification.  This includes both the 'EnvironmentVariables' property for this tool spec, and any build options which define that their value should be exported via their 'SetValueInEnvironmentVariable' property.
@@ -1313,7 +1323,7 @@ open class CommandLineToolSpec : PropertyDomainSpec, SpecType, TaskTypeDescripti
 
         // Add environment variables from build options which specify they should be added via a 'SetValueInEnvironmentVariable' property.
         // FIXME: Optimize the list to search here.
-        for buildOption in self.flattenedOrderedBuildOptions {
+        for buildOption in cbc.producer.effectiveFlattenedOrderedBuildOptions(self) {
             if let assignment = buildOption.getEnvironmentAssignmentForCommand(cbc, lookup: wrappedLookup) {
                 environment.append(assignment)
             }

Sources/SWBCore/SpecImplementations/PropertyDomainSpec.swift

@@ -343,7 +343,7 @@ private let buildOptionTypes: [String: any BuildOptionType] = [
     let inputInclusions: [MacroStringExpression]?
 
     /// Additional output dependencies to consider when this build option is active.
-    let outputDependencies: [MacroStringExpression]?
+    let outputDependencies: [(path: MacroStringExpression, fileType: MacroStringExpression?)]?
 
     /// Helper function for extract an individual value definition from a dictionary.
     private static func parseBuildOptionValue(_ parser: SpecParser, _ name: String, _ type: any BuildOptionType, _ data: [String: PropertyListItem]) -> (String, BuildOptionValue)? {
@@ -804,7 +804,7 @@ private let buildOptionTypes: [String: any BuildOptionType] = [
         var dependencyFormat: DependencyDataFormat? = nil
         var featureFlags: [String]? = nil
         var inputInclusions: [MacroStringExpression]? = nil
-        var outputDependencies: [MacroStringExpression]? = nil
+        var outputDependencies: [(path: MacroStringExpression, fileType: MacroStringExpression?)]? = nil
         for (key, valueData) in items {
             switch key {
             case "Name":
@@ -1031,14 +1031,36 @@ private let buildOptionTypes: [String: any BuildOptionType] = [
             case "OutputDependencies":
                 switch valueData {
                 case .plString(let value):
-                    outputDependencies = [parser.delegate.internalMacroNamespace.parseString(value)]
+                    outputDependencies = [(parser.delegate.internalMacroNamespace.parseString(value), nil)]
                 case .plArray(let values):
                     outputDependencies = values.compactMap({ data in
-                        guard case .plString(let value) = data else {
-                            error("expected all string values in array for '\(key)'")
+                        switch data {
+                        case let .plString(value):
+                            return (parser.delegate.internalMacroNamespace.parseString(value), nil)
+                        case let .plDict(value):
+                            let path: MacroStringExpression
+                            switch value["Path"] {
+                            case let .plString(expr):
+                                path = parser.delegate.internalMacroNamespace.parseString(expr)
+                            default:
+                                error("expected string value for subkey 'Path' in element of array in '\(key)'")
+                                return nil
+                            }
+
+                            let fileType: MacroStringExpression
+                            switch value["FileType"] {
+                            case let .plString(expr):
+                                fileType = parser.delegate.internalMacroNamespace.parseString(expr)
+                            default:
+                                error("expected string value for subkey 'FileType' in element of array in '\(key)'")
+                                return nil
+                            }
+
+                            return (path, fileType)
+                        default:
+                            error("expected all string or dictionary values in array for '\(key)'")
                             return nil
                         }
-                        return parser.delegate.internalMacroNamespace.parseString(value)
                     })
                 default:
                     error("expected string or array value for build option key '\(key)'")
@@ -1720,11 +1742,11 @@ open class PropertyDomainSpec : Spec, @unchecked Sendable {
 
 
 /// Extensions to PropertyDomainSpec for performance testing.
-public extension PropertyDomainSpec {
+extension PropertyDomainSpec {
 
     /// Creates and returns a ``MacroValueAssignmentTable`` populated with the default values of the receiver's build options.
     /// The table's namespace is also returned so that the caller can add further settings to it if desired.
-    func macroTableForBuildOptionDefaults(_ core: Core) -> (MacroValueAssignmentTable, MacroNamespace) {
+    @_spi(Testing) public func macroTableForBuildOptionDefaults(_ core: Core) -> (MacroValueAssignmentTable, MacroNamespace) {
         var table = MacroValueAssignmentTable(namespace: core.specRegistry.internalMacroNamespace)
         for option in self.flattenedOrderedBuildOptions {
             guard let value = option.defaultValue else { continue }