Add String Catalog symbol generation support to SwiftBuild (#582)

* Add String Catalog symbol generation support to SwiftBuild

* Mark xcstrings as a type producing Swift sources

Author: matthewseaman

Sources/SWBApplePlatform/Plugin.swift

@@ -202,7 +202,7 @@ struct XCStringsInputFileGroupingStrategyExtension: InputFileGroupingStrategyExt
     }
 
     func fileTypesCompilingToSwiftSources() -> [String] {
-        return []
+        return ["text.json.xcstrings"]
     }
 }
 

Sources/SWBApplePlatform/XCStringsCompiler.swift

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 import SWBUtil
+import SWBMacro
 public import SWBCore
 import Foundation
 
@@ -47,6 +48,147 @@ public final class XCStringsCompilerSpec: GenericCompilerSpec, SpecIdentifierTyp
             return
         }
 
+        if shouldGenerateSymbols(cbc) {
+            constructSymbolGenerationTask(cbc, delegate)
+        }
+
+        if shouldCompileCatalog(cbc) {
+            await constructCatalogCompilationTask(cbc, delegate)
+        }
+    }
+
+    public override var supportsInstallHeaders: Bool {
+        // Yes but we will only perform symbol generation in that case.
+        return true
+    }
+
+    public override var supportsInstallAPI: Bool {
+        // Yes but we will only perform symbol generation in that case.
+        // This matches Asset Catalog symbol generation in order to workaround an issue with header whitespace.
+        // rdar://106447203 (Symbols: Enabling symbols for IB causes installapi failure)
+        return true
+    }
+
+    /// Whether we should generate tasks to generate code symbols for strings.
+    private func shouldGenerateSymbols(_ cbc: CommandBuildContext) -> Bool {
+        guard cbc.scope.evaluate(BuiltinMacros.STRING_CATALOG_GENERATE_SYMBOLS) else {
+            return false
+        }
+
+        // Yes for standard builds/installs as well as headers/api and exportloc (which includes headers).
+        // No for installloc.
+        let buildComponents = cbc.scope.evaluate(BuiltinMacros.BUILD_COMPONENTS)
+        guard buildComponents.contains("build") || buildComponents.contains("headers") || buildComponents.contains("api") else {
+            return false
+        }
+
+        // Avoid symbol generation for xcstrings inside variant groups because that implies association with a resource such as a xib.
+        guard cbc.input.regionVariantName == nil else {
+            return false
+        }
+
+        // We are only supporting Swift symbols at the moment so don't even generate the task if there are not Swift sources.
+        // If this is a synthesized Package resource target, we won't have Swift sources either.
+        // That's good since the symbol gen will happen for the code target instead.
+        let targetContainsSwiftSources = (cbc.producer.configuredTarget?.target as? StandardTarget)?.sourcesBuildPhase?.containsSwiftSources(cbc.producer, cbc.producer, cbc.scope, cbc.producer.filePathResolver) ?? false
+        guard targetContainsSwiftSources else {
+            return false
+        }
+
+        return true
+    }
+
+    /// Whether we should generate tasks to compile the .xcstrings file to .strings/dict files.
+    private func shouldCompileCatalog(_ cbc: CommandBuildContext) -> Bool {
+        // Yes for standard builds/installs and installloc.
+        // No for exportloc and headers/api.
+        let buildComponents = cbc.scope.evaluate(BuiltinMacros.BUILD_COMPONENTS)
+        guard buildComponents.contains("build") || buildComponents.contains("installLoc") else {
+            return false
+        }
+
+        // If this is a Package target with a synthesized resource target, compile the catalog with the resources instead of here.
+        let isMainPackageWithResourceBundle = !cbc.scope.evaluate(BuiltinMacros.PACKAGE_RESOURCE_BUNDLE_NAME).isEmpty
+        return !isMainPackageWithResourceBundle
+    }
+
+    private struct SymbolGenPayload: TaskPayload {
+
+        let effectivePlatformName: String
+
+        init(effectivePlatformName: String) {
+            self.effectivePlatformName = effectivePlatformName
+        }
+
+        func serialize<T>(to serializer: T) where T : SWBUtil.Serializer {
+            serializer.serializeAggregate(1) {
+                serializer.serialize(effectivePlatformName)
+            }
+        }
+
+        init(from deserializer: any SWBUtil.Deserializer) throws {
+            try deserializer.beginAggregate(1)
+            self.effectivePlatformName = try deserializer.deserialize()
+        }
+
+    }
+
+    public override var payloadType: (any TaskPayload.Type)? {
+        return SymbolGenPayload.self
+    }
+
+    /// Generates a task for generating code symbols for strings.
+    private func constructSymbolGenerationTask(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) {
+        // The template spec file contains fields suitable for the compilation step.
+        // But here we construct a custom command line for symbol generation.
+        let execPath = resolveExecutablePath(cbc, Path("xcstringstool"))
+        var commandLine = [execPath.str, "generate-symbols"]
+
+        // For now shouldGenerateSymbols only returns true if there are Swift sources.
+        // So we only generate Swift symbols for now.
+        commandLine.append(contentsOf: ["--language", "swift"])
+
+        let outputDir = cbc.scope.evaluate(BuiltinMacros.DERIVED_SOURCES_DIR)
+        commandLine.append(contentsOf: ["--output-directory", outputDir.str])
+
+        // Input file
+        let inputPath = cbc.input.absolutePath
+        commandLine.append(inputPath.str)
+
+        let outputPaths = [
+            "GeneratedStringSymbols_\(inputPath.basenameWithoutSuffix).swift"
+        ]
+        .map { fileName in
+            return outputDir.join(fileName)
+        }
+
+        for output in outputPaths {
+            delegate.declareOutput(FileToBuild(absolutePath: output, inferringTypeUsing: cbc.producer))
+        }
+
+        // Use just first path for now since we're not even sure if we'll support languages beyond Swift.
+        let ruleInfo = ["GenerateStringSymbols", outputPaths.first!.str, inputPath.str]
+        let execDescription = "Generate symbols for \(inputPath.basename)"
+
+        let payload = SymbolGenPayload(effectivePlatformName: LocalizationBuildPortion.effectivePlatformName(scope: cbc.scope, sdkVariant: cbc.producer.sdkVariant))
+
+        delegate.createTask(
+            type: self,
+            payload: payload,
+            ruleInfo: ruleInfo,
+            commandLine: commandLine,
+            environment: environmentFromSpec(cbc, delegate),
+            workingDirectory: cbc.producer.defaultWorkingDirectory,
+            inputs: [inputPath],
+            outputs: outputPaths,
+            execDescription: execDescription,
+            preparesForIndexing: true,
+            enableSandboxing: enableSandboxing
+        )
+    }
+
+    /// Generates a task for compiling the .xcstrings to .strings/dict files.
+    private func constructCatalogCompilationTask(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) async {
         let commandLine = await commandLineFromTemplate(cbc, delegate, optionContext: discoveredCommandLineToolSpecInfo(cbc.producer, cbc.scope, delegate)).map(\.asString)
 
         // We can't know our precise outputs statically because we don't know what languages are in the xcstrings file,
@@ -75,7 +217,17 @@ public final class XCStringsCompilerSpec: GenericCompilerSpec, SpecIdentifierTyp
         }
 
         if !outputs.isEmpty {
-            delegate.createTask(type: self, ruleInfo: defaultRuleInfo(cbc, delegate), commandLine: commandLine, environment: environmentFromSpec(cbc, delegate), workingDirectory: cbc.producer.defaultWorkingDirectory, inputs: [cbc.input.absolutePath], outputs: outputs, execDescription: resolveExecutionDescription(cbc, delegate), enableSandboxing: enableSandboxing)
+            delegate.createTask(
+                type: self,
+                ruleInfo: defaultRuleInfo(cbc, delegate),
+                commandLine: commandLine,
+                environment: environmentFromSpec(cbc, delegate),
+                workingDirectory: cbc.producer.defaultWorkingDirectory,
+                inputs: [cbc.input.absolutePath],
+                outputs: outputs,
+                execDescription: resolveExecutionDescription(cbc, delegate),
+                enableSandboxing: enableSandboxing
+            )
         } else {
             // If there won't be any outputs, there's no reason to run the compiler.
             // However, we still need to leave some indication in the build graph that there was a compilable xcstrings file here so that generateLocalizationInfo can discover it.
@@ -131,8 +283,7 @@ public final class XCStringsCompilerSpec: GenericCompilerSpec, SpecIdentifierTyp
     }
 
     public override func generateLocalizationInfo(for task: any ExecutableTask, input: TaskGenerateLocalizationInfoInput) -> [TaskGenerateLocalizationInfoOutput] {
-        // Tell the build system about the xcstrings file we took as input.
-        // No need to use a TaskPayload for this because the only data we need is input path, which is already stored on the Task.
+        // Tell the build system about the xcstrings file we took as input, as well as any generated symbol files.
 
         // These asserts just check to make sure the broader implementation hasn't changed since we wrote this method,
         // in case something here would need to change.
@@ -142,7 +293,18 @@ public final class XCStringsCompilerSpec: GenericCompilerSpec, SpecIdentifierTyp
 
         // Our input paths are .xcstrings (only expecting 1).
         // NOTE: We also take same-named .strings/dict files as input, but those are only used to diagnose errors and when they exist we fail before we ever generate the task.
-        return [TaskGenerateLocalizationInfoOutput(compilableXCStringsPaths: task.inputPaths)]
+        var infos = [TaskGenerateLocalizationInfoOutput(compilableXCStringsPaths: task.inputPaths)]
+
+        if let payload = task.payload as? SymbolGenPayload,
+           let xcstringsPath = task.inputPaths.only {
+            let generatedSourceFiles = task.outputPaths.filter { $0.fileExtension == "swift" }
+            var info = TaskGenerateLocalizationInfoOutput()
+            info.effectivePlatformName = payload.effectivePlatformName
+            info.generatedSymbolFilesByXCStringsPath = [xcstringsPath: generatedSourceFiles]
+            infos.append(info)
+        }
+
+        return infos
     }
 
 }

Sources/SWBBuildService/LocalizationInfo.swift

@@ -41,6 +41,15 @@ struct LocalizationInfoOutput {
 
     /// Paths to .stringsdata files produced by this target, grouped by build attributes such as platform and architecture.
     fileprivate(set) var producedStringsdataPaths: [LocalizationBuildPortion: Set<Path>] = [:]
+
+    /// The name of the primary platform we were building for.
+    ///
+    /// Mac Catalyst is treated as its own platform.
+    fileprivate(set) var effectivePlatformName: String?
+
+    /// Paths to generated source code files holding string symbols, keyed by xcstrings file path.
+    fileprivate(set) var generatedSymbolFilesByXCStringsPath = [Path: Set<Path>]()
+
 }
 
 extension BuildDescriptionManager {
@@ -98,9 +107,24 @@ extension BuildDescription {
                 .reduce([:], { aggregate, partial in aggregate.merging(partial, uniquingKeysWith: +) })
                 .mapValues { Set($0) }
 
+            // Only really expecting to have one platform for a given build.
+            // So just use the first seen one as primary.
+            let effectivePlatformName = taskLocalizationOutputs.compactMap(\.effectivePlatformName).first
+
             outputsByTarget[targetGUID, default: LocalizationInfoOutput(targetIdentifier: targetGUID)]
                 .compilableXCStringsPaths.formUnion(taskXCStringsPaths)
             outputsByTarget[targetGUID]?.producedStringsdataPaths.merge(taskStringsdataPaths, uniquingKeysWith: { $0.union($1) })
+
+            if outputsByTarget[targetGUID]?.effectivePlatformName == nil && effectivePlatformName != nil {
+                outputsByTarget[targetGUID]?.effectivePlatformName = effectivePlatformName
+            }
+
+            let taskGeneratedSymbolFiles = taskLocalizationOutputs
+                .map(\.generatedSymbolFilesByXCStringsPath)
+                .reduce([:], { aggregate, partial in aggregate.merging(partial, uniquingKeysWith: +) })
+                .mapValues { Set($0) }
+
+            outputsByTarget[targetGUID]?.generatedSymbolFilesByXCStringsPath.merge(taskGeneratedSymbolFiles, uniquingKeysWith: { $0.union($1) })
         }
 
         return Array(outputsByTarget.values)

Sources/SWBBuildService/Messages.swift

@@ -901,7 +901,9 @@ private struct GetLocalizationInfoMsg: MessageHandler {
                     for (buildPortion, paths) in infoOutput.producedStringsdataPaths {
                         stringsdataPaths[LocalizationInfoBuildPortion(effectivePlatformName: buildPortion.effectivePlatformName, variant: buildPortion.variant, architecture: buildPortion.architecture)] = paths
                     }
-                    return LocalizationInfoMessagePayload(targetIdentifier: infoOutput.targetIdentifier, compilableXCStringsPaths: infoOutput.compilableXCStringsPaths, producedStringsdataPaths: stringsdataPaths)
+                    var payload = LocalizationInfoMessagePayload(targetIdentifier: infoOutput.targetIdentifier, compilableXCStringsPaths: infoOutput.compilableXCStringsPaths, producedStringsdataPaths: stringsdataPaths, effectivePlatformName: infoOutput.effectivePlatformName)
+                    payload.generatedSymbolFilesByXCStringsPath = infoOutput.generatedSymbolFilesByXCStringsPath
+                    return payload
                 }))
                 return response
             } catch {

Sources/SWBCore/Settings/BuiltinMacros.swift

@@ -979,6 +979,7 @@ public final class BuiltinMacros {
     public static let SHALLOW_BUNDLE = BuiltinMacros.declareBooleanMacro("SHALLOW_BUNDLE")
     public static let SHARED_FRAMEWORKS_FOLDER_PATH = BuiltinMacros.declarePathMacro("SHARED_FRAMEWORKS_FOLDER_PATH")
     public static let SHARED_SUPPORT_FOLDER_PATH = BuiltinMacros.declarePathMacro("SHARED_SUPPORT_FOLDER_PATH")
+    public static let STRING_CATALOG_GENERATE_SYMBOLS = BuiltinMacros.declareBooleanMacro("STRING_CATALOG_GENERATE_SYMBOLS")
     public static let STRINGS_FILE_INPUT_ENCODING = BuiltinMacros.declareStringMacro("STRINGS_FILE_INPUT_ENCODING")
     public static let STRINGS_FILE_OUTPUT_ENCODING = BuiltinMacros.declareStringMacro("STRINGS_FILE_OUTPUT_ENCODING")
     public static let STRINGS_FILE_OUTPUT_FILENAME = BuiltinMacros.declareStringMacro("STRINGS_FILE_OUTPUT_FILENAME")
@@ -2133,6 +2134,7 @@ public final class BuiltinMacros {
         SOURCE_ROOT,
         SPECIALIZATION_SDK_OPTIONS,
         SRCROOT,
+        STRING_CATALOG_GENERATE_SYMBOLS,
         STRINGSDATA_DIR,
         STRINGS_FILE_INPUT_ENCODING,
         STRINGS_FILE_OUTPUT_ENCODING,

Sources/SWBCore/Settings/Settings.swift

@@ -4153,6 +4153,7 @@ private class SettingsBuilder {
         if let project, project.isPackage, project.developmentRegion != nil {
             table.push(BuiltinMacros.LOCALIZATION_EXPORT_SUPPORTED, literal: true)
             table.push(BuiltinMacros.SWIFT_EMIT_LOC_STRINGS, literal: true)
+            table.push(BuiltinMacros.STRING_CATALOG_GENERATE_SYMBOLS, literal: true)
         }
 
         return table

Sources/SWBCore/TaskGeneration.swift

@@ -1163,12 +1163,21 @@ public struct TaskGenerateLocalizationInfoOutput {
     /// Paths to .stringsdata files produced by this task, grouped by build attributes such as platform and architecture.
     public let producedStringsdataPaths: [LocalizationBuildPortion: [Path]]
 
+    /// The name of the primary platform we were building for.
+    ///
+    /// Mac Catalyst is treated as its own platform.
+    public var effectivePlatformName: String?
+
+    /// Paths to generated source code files holding string symbols, keyed by xcstrings file path.
+    public var generatedSymbolFilesByXCStringsPath = [Path: [Path]]()
+
     /// Create output to describe some portion of localization info for a Task.
     ///
     /// - Parameters:
     ///   - compilableXCStringsPaths: Paths to input source .xcstrings files.
     ///   - producedStringsdataPaths: Paths to output .stringsdata files.
-    public init(compilableXCStringsPaths: [Path] = [], producedStringsdataPaths: [LocalizationBuildPortion: [Path]] = [:]) {
+    public init(compilableXCStringsPaths: [Path] = [],
+                producedStringsdataPaths: [LocalizationBuildPortion: [Path]] = [:]) {
         self.compilableXCStringsPaths = compilableXCStringsPaths
         self.producedStringsdataPaths = producedStringsdataPaths
     }

Sources/SWBProjectModel/PIFGenerationModel.swift

@@ -1128,6 +1128,11 @@ public struct SwiftBuildFileType: Sendable {
         fileTypeIdentifier: "folder.abstractassetcatalog"
     )
 
+    public static let xcstrings: SwiftBuildFileType = SwiftBuildFileType(
+        fileType: "xcstrings",
+        fileTypeIdentifier: "text.json.xcstrings"
+    )
+
     public static let xcdatamodeld: SwiftBuildFileType = SwiftBuildFileType(
         fileType: "xcdatamodeld",
         fileTypeIdentifier: "wrapper.xcdatamodeld"
@@ -1165,6 +1170,7 @@ public struct SwiftBuildFileType: Sendable {
 
     public static let all: [SwiftBuildFileType] = [
         .xcassets,
+        .xcstrings,
         .xcdatamodeld,
         .xcdatamodel,
         .xcmappingmodel,

Sources/SWBProtocol/MessageSupport.swift

@@ -542,9 +542,21 @@ public struct LocalizationInfoMessagePayload: SerializableCodable, Equatable, Se
     /// Paths to .stringsdata files produced by this target, grouped by build attributes such as platform and architecture.
     public let producedStringsdataPaths: [LocalizationInfoBuildPortion: Set<Path>]
 
-    public init(targetIdentifier: String, compilableXCStringsPaths: Set<Path>, producedStringsdataPaths: [LocalizationInfoBuildPortion: Set<Path>]) {
+    /// The name of the primary platform we were building for.
+    ///
+    /// Mac Catalyst is treated as its own platform.
+    public let effectivePlatformName: String?
+
+    /// Paths to generated source code files holding string symbols, keyed by xcstrings file path.
+    public var generatedSymbolFilesByXCStringsPath = [Path: Set<Path>]()
+
+    public init(targetIdentifier: String,
+                compilableXCStringsPaths: Set<Path>,
+                producedStringsdataPaths: [LocalizationInfoBuildPortion: Set<Path>],
+                effectivePlatformName: String?) {
         self.targetIdentifier = targetIdentifier
         self.compilableXCStringsPaths = compilableXCStringsPaths
         self.producedStringsdataPaths = producedStringsdataPaths
+        self.effectivePlatformName = effectivePlatformName
     }
 }