Generate symbol graph data for command-line tool info

cmcgee1024 · cmcgee1024 · commit 615331442a4a · 2025-11-05T11:42:27.000-05:00
Make any target documentable if it contains a DocC catalog
diff --git a/Package.swift b/Package.swift
@@ -575,6 +575,7 @@ let package = Package(
             dependencies: [
                 .product(name: "ArgumentParser", package: "swift-argument-parser"),
                 .product(name: "OrderedCollections", package: "swift-collections"),
+                .product(name: "SymbolKit", package: "swift-docc-symbolkit"),
                 "Basics",
                 "BinarySymbols",
                 "Build",
@@ -1110,6 +1111,7 @@ if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil {
         .package(url: "https://github.com/apple/swift-argument-parser.git", .upToNextMinor(from: "1.5.1")),
         .package(url: "https://github.com/apple/swift-crypto.git", .upToNextMinor(from: "3.0.0")),
         .package(url: "https://github.com/swiftlang/swift-syntax.git", branch: relatedDependenciesBranch),
+        .package(url: "https://github.com/swiftlang/swift-docc-symbolkit.git", branch: relatedDependenciesBranch),
         .package(url: "https://github.com/apple/swift-system.git", from: "1.1.1"),
         .package(url: "https://github.com/apple/swift-collections.git", "1.0.1" ..< "1.2.0"),
         .package(url: "https://github.com/apple/swift-certificates.git", "1.0.1" ..< "1.6.0"),
@@ -1163,7 +1165,8 @@ if !shoudUseSwiftBuildFramework {
 
     if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil {
         package.dependencies += [
-            .package(url: "https://github.com/swiftlang/swift-build.git", branch: relatedDependenciesBranch),
+            //.package(url: "https://github.com/swiftlang/swift-build.git", branch: relatedDependenciesBranch),
+            .package(path: "../swift-build"),
         ]
     } else {
         package.dependencies += [
diff --git a/Sources/Commands/PackageCommands/GenerateDocumentation.swift b/Sources/Commands/PackageCommands/GenerateDocumentation.swift
@@ -19,93 +19,60 @@ import PackageGraph
 import Workspace
 import SPMBuildCore
 import ArgumentParserToolInfo
+import SymbolKit
 
 extension CommandInfoV0 {
-  var doccReferenceFileName: String {
-    doccReferenceTitle + ".md"
+  func toSymbolGraph() -> SymbolGraph {
+    return SymbolGraph(
+      metadata: SymbolGraph.Metadata(formatVersion: .init(major: 0, minor: 6, patch: 0), generator: "SwiftPM"),
+      module: SymbolGraph.Module(name: self.commandName, platform: .init(architecture: "arm64", vendor: nil, operatingSystem: .init(name: "macOS"), environment: nil)),
+      symbols: toSymbols(),
+      relationships: []
+    )
   }
 
-  var doccReferenceDocumentTitle: String {
-    let parts = (superCommands ?? []) + [commandName]
-    return parts.joined(separator: ".").uppercased()
-  }
-
-  var doccReferenceTitle: String {
-    let parts = (superCommands ?? []) + [commandName]
-    return parts.joined(separator: ".")
-  }
+  func toSymbols(_ path: [String] = []) -> [SymbolGraph.Symbol] {
+    var symbols: [SymbolGraph.Symbol] = []
 
-  var doccReferenceName: String {
-    let parts = (superCommands ?? []) + [commandName]
-    return parts.joined(separator: " ")
-  }
-}
+    var myPath = path
+    myPath.append(self.commandName)
 
-extension CommandInfoV0 {
-  /// Recursively parses a command to generate markdown content that describes the command.
-  /// - Parameters:
-  ///   - path: The path of subcommands from the root command.
-  ///   - markdownStyle: The flavor of markdown to emit, either `docc` or `github`
-  /// - Returns: A multi-line markdown file that describes the command.
-  ///
-  /// If `path` is empty, it represents a top-level command.
-  /// Otherwise it's a subcommand, potentially recursive to multiple levels.
-  func toMarkdown(_ path: [String]) -> String {
-    var result =
-      String(repeating: "#", count: path.count + 1)
-      + " \(self.doccReferenceTitle)\n\n"
-
-    // sets the max width for generating code blocks of content based
-    // on the style
-    let blockWrapLength: Int = 60
-
-    if path.count == 0 {
-      result += "<!-- Generated by swift-argument-parser -->\n\n"
+    guard myPath.last != "help" else {
+      return []
     }
 
-    if let abstract = self.abstract {
-      result += "\(abstract)\n\n"
-    }
+    var line = 0
+    var docComments: SymbolGraph.LineList = if let abstract = self.abstract { .init([SymbolGraph.LineList.Line(text: abstract, range: nil )]) } else { .init([]) }
+    line += 2
 
     if let args = self.arguments, args.count != 0 {
-      result += "```\n"
-      let commandString = (path + [self.commandName]).joined(separator: " ")
-      result +=
-        commandString
-        + self.usage(
-          startlength: commandString.count, wraplength: blockWrapLength)
-      result += "\n```\n\n"
-    }
+      let commandString: String = myPath.joined(separator: " ")
 
-    if let discussion = self.discussion {
-      result += "\(discussion)\n\n"
+      docComments = .init(docComments.lines + [SymbolGraph.LineList.Line(text: "```\n" + commandString + self.usage(startlength: commandString.count, wraplength: 60) + "\n```", range: nil )]) // TODO parameterize the wrap length
+      line += 2
     }
 
-    if let args = self.arguments {
-      for arg in args {
-        guard arg.shouldDisplay else {
-          continue
-        }
-
-        result += "- term **\(arg.identity())**:\n\n"
-
-        if let abstract = arg.abstract {
-          result += "*\(abstract)*\n\n"
-        }
-        if let discussion = arg.discussion {
-          result += discussion + "\n\n"
-        }
-        result += "\n"
-      }
+    if let discussion = self.discussion {
+      docComments = .init(docComments.lines + (discussion.split(separator: "\n").map({ SymbolGraph.LineList.Line(text: String($0), range: nil )})))
+      line += 2
     }
 
-    for subcommand in self.subcommands ?? [] {
-      result +=
-        subcommand.toMarkdown(
-          path + [self.commandName]) + "\n\n"
+    // TODO: Maybe someday there will be command-line semantics for the symbols and then these can be declared with more sensible categories
+    symbols.append(SymbolGraph.Symbol(
+      identifier: .init(precise: "s:\(myPath.joined(separator: " "))", interfaceLanguage: "swift"),
+      names: .init(title: self.commandName, navigator: nil, subHeading: nil, prose: nil),
+      pathComponents: myPath,
+      docComment: docComments,
+      accessLevel: SymbolGraph.Symbol.AccessControl(rawValue: "public"),
+      kind: SymbolGraph.Symbol.Kind(parsedIdentifier: .`func`, displayName: "command"),
+      mixins: [:]
+    ))
+
+    for cmd in self.subcommands ?? [] {
+      symbols.append(contentsOf: cmd.toSymbols(myPath))
     }
 
-    return result
+    return symbols
   }
 
   /// Returns a mutl-line string that presents the arguments for a command.
@@ -221,9 +188,10 @@ struct GenerateDocumentation: AsyncSwiftCommand {
     var globalOptions: GlobalOptions
 
     func run(_ swiftCommandState: SwiftCommandState) async throws {
+        // TODO someday we might be able to populate the landing page with details about the package as a whole, such as traits, or even a DocC catalog that covers package-level topics
+
         let buildSystem = try await swiftCommandState.createBuildSystem()
 
-        // TODO build only the product related targets when not generating internal docs
         let outputs = try await buildSystem.build(subset: .allExcludingTests, buildOutputs: [
             .symbolGraph(
                 .init(
@@ -267,105 +235,110 @@ struct GenerateDocumentation: AsyncSwiftCommand {
                 }
             } else {
                 modules.append(contentsOf: rootPackage.modules)
+                products.append(contentsOf: rootPackage.products)
             }
         }
 
         for product in products {
           if product.type == .executable {
-              let exec = builtArtifacts.filter({ $0.1.kind == .executable && $0.0 == "\(product.name)-product" }).first?.1.path
-
-              guard let exec else {
-                  print("Couldn't find \(product.name)")
-                  continue
+              let doccCatalogDir = product.modules.first?.underlying.others.filter({ $0.extension?.lowercased() == "docc" }).first
+              var symbolGraphDir: AbsolutePath? = nil
+
+              if let exec = builtArtifacts.filter({ $0.1.kind == .executable && $0.0 == "\(product.name)-product" }).first?.1.path {
+              do {
+                  // FIXME run the executable within a very restricted sandbox
+                  let dumpHelpProcess = AsyncProcess(args: [exec, "--experimental-dump-help"], outputRedirection: .collect)
+                  try dumpHelpProcess.launch()
+                  let result = try await dumpHelpProcess.waitUntilExit()
+                  let output = try result.utf8Output()
+                  let toolInfo = try JSONDecoder().decode(ToolInfoV0.self, from: output)
+
+                  // Creating a symbol graph that represents the command-line structure
+                  symbolGraphDir = buildPath.appending(components: ["tool-symbol-graph", product.name])
+                  guard let graphDir = symbolGraphDir else {fatalError()}
+
+                  try? swiftCommandState.fileSystem.removeFileTree(graphDir)
+                  try swiftCommandState.fileSystem.createDirectory(graphDir, recursive: true)
+                  
+                  let graph = toolInfo.command.toSymbolGraph()
+                  let doc = try JSONEncoder().encode(graph)
+                  let graphFile = graphDir.appending(components: ["\(product.name).symbols.json"])
+                  try swiftCommandState.fileSystem.writeFileContents(graphFile, data: doc)
+              } catch {
+                  print("warning: could not generate tool info documentation for \(product.name)")
               }
-
-               do {
-                    // FIXME run the executable within a very restricted sandbox
-                    let dumpHelpProcess = AsyncProcess(args: [exec, "--experimental-dump-help"], outputRedirection: .collect)
-                    try dumpHelpProcess.launch()
-                    let result = try await dumpHelpProcess.waitUntilExit()
-                    let output = try result.utf8Output()
-                    let toolInfo = try JSONDecoder().decode(ToolInfoV0.self, from: output)
-                    
-                    let page = toolInfo.command.toMarkdown([])
-
-                    // Creating a simple DocC catalog
-                    // TODO copy over an existing DocC catalog for this module if one exists already
-                    let catalogDir = buildPath.appending(components: ["tool-docc-catalog", product.name, "Documentation.docc"])
-                    try? swiftCommandState.fileSystem.removeFileTree(catalogDir)
-                    try swiftCommandState.fileSystem.createDirectory(catalogDir, recursive: true)
-                    let summaryMd = catalogDir.appending(components: ["\(product.name.lowercased()).md"])
-                    try swiftCommandState.fileSystem.writeFileContents(summaryMd, string: page)
-
-                    let archiveDir = buildPath.appending(components: ["tool-docc-archive", "\(product.name).doccarchive"])
-                    try? swiftCommandState.fileSystem.removeFileTree(archiveDir)
-                    try swiftCommandState.fileSystem.createDirectory(archiveDir.parentDirectory, recursive: true)
-
-                    print("CONVERT TOOL: \(product.name)")
-            
-                    let process = try Process.run(URL(fileURLWithPath: doccExecutable.pathString), arguments: [
-                        "convert",
-                        catalogDir.pathString,
-                        "--fallback-display-name=\(product.name)",
-                        "--fallback-bundle-identifier=\(product.name)",
-                        "--output-path=\(archiveDir)",
-                    ])
-                    process.waitUntilExit()
-
-                    if swiftCommandState.fileSystem.exists(archiveDir) {
-                        doccArchives.append(archiveDir.pathString)
-                    }
-                } catch {
-                    print("warning: could not generate tool info documentation for \(product.name)")
-                }
-                
-                continue
             }
-        }
 
-        for module: ResolvedModule in modules {
-            guard module.type != .test && module.type != .plugin && module.type != .executable else {
-                continue
+            guard doccCatalogDir != nil || symbolGraphDir != nil else {
+              print("Skipping \(product.name) because there is no DocC catalog and there is no symbol graph that could be generated for it.")
+              continue
             }
 
-            let inputPathDir = symbolGraph.outputLocationForTarget(module.name, try swiftCommandState.productsBuildParameters)
-            let inputPath = buildPath.appending(components: inputPathDir)
+            let catalogArgs = if let doccCatalogDir  {[doccCatalogDir.pathString]} else {[String]()}
+            let graphArgs = if let symbolGraphDir {["--additional-symbol-graph-dir=\(symbolGraphDir)"]} else {[String]()}
 
-            guard swiftCommandState.fileSystem.exists(inputPath) else {
-              continue
+            print("CONVERTING: \(product.name)")
+
+            let archiveDir = buildPath.appending(components: ["tool-docc-archive", "\(product.name).doccarchive"])
+            try? swiftCommandState.fileSystem.removeFileTree(archiveDir)
+            try swiftCommandState.fileSystem.createDirectory(archiveDir.parentDirectory, recursive: true)
+    
+            let process = try Process.run(URL(fileURLWithPath: doccExecutable.pathString), arguments: [
+                "convert",
+                ] + catalogArgs + [
+                "--fallback-display-name=\(product.name)",
+                "--fallback-bundle-identifier=\(product.name)",
+                ] + graphArgs + [
+                "--output-path=\(archiveDir)",
+            ])
+            process.waitUntilExit()
+
+            if swiftCommandState.fileSystem.exists(archiveDir) {
+                print("SUCCESS!")
+                doccArchives.append(archiveDir.pathString)
             }
+          }
+        }
 
-            let outputPathDir = [String](inputPathDir.dropLast()) + [inputPathDir.last!.replacing(".symbolgraphs", with: ".doccarchive")]
-            let outputPath = buildPath.appending(components: outputPathDir)
+        for module: ResolvedModule in modules {
+            let symbolGraphDir = symbolGraph.outputLocationForTarget(module.name, try swiftCommandState.productsBuildParameters)
+            let symbolGraphPath = buildPath.appending(components: symbolGraphDir)
 
             // The DocC catalog for this module is any directory with the docc file extension
-            let doccCatalog = module.underlying.others.first { sourceFile in
+            let doccCatalogDir = module.underlying.others.first { sourceFile in
                 return sourceFile.extension?.lowercased() == "docc"
             }
 
-            let catalogArgs = if let doccCatalog  {[doccCatalog.pathString]} else {[String]()}
+            guard doccCatalogDir != nil || swiftCommandState.fileSystem.exists(symbolGraphPath) else {
+              print("Skipping \(module.name) because there is no DocC catalog and there is no symbol graph that could be generated for it.")
+              continue
+            }
 
-            print("CONVERT: \(module.name)")
+            let catalogArgs = if let doccCatalogDir {[doccCatalogDir.pathString]} else {[String]()}
+            let graphArgs = if swiftCommandState.fileSystem.exists(symbolGraphPath) {["--additional-symbol-graph-dir=\(symbolGraphPath)"]} else {[String]()}
+
+            print("CONVERTING: \(module.name)")
+
+            let archiveDir = buildPath.appending(components: ["module-docc-archive", "\(module.name).doccarchive"])
+            try? swiftCommandState.fileSystem.removeFileTree(archiveDir)
+            try swiftCommandState.fileSystem.createDirectory(archiveDir.parentDirectory, recursive: true)
 
-            print("docc convert \(catalogArgs.joined(separator: " ")) \(["--fallback-display-name=\(module.name)", "--fallback-bundle-identifier=\(module.name)", "--additional-symbol-graph-dir=\(inputPath)", "--output-path=\(outputPath)"].joined(separator: " "))")
-            
             let process = try Process.run(URL(fileURLWithPath: doccExecutable.pathString), arguments: [
                 "convert",
                 ] + catalogArgs + [
                 "--fallback-display-name=\(module.name)",
                 "--fallback-bundle-identifier=\(module.name)",
-                "--additional-symbol-graph-dir=\(inputPath)",
-                "--output-path=\(outputPath)",
+                ] + graphArgs + [
+                "--output-path=\(archiveDir)",
             ])
             process.waitUntilExit()
 
-            if swiftCommandState.fileSystem.exists(outputPath) {
-                doccArchives.append(outputPath.pathString)
+            if swiftCommandState.fileSystem.exists(archiveDir) {
+                doccArchives.append(archiveDir.pathString)
             }
         }
 
         guard doccArchives.count > 0 else {
-            // FIXME consider presenting just the README.md contents if possible
             print("No modules are available to document.")
             return
         }
@@ -376,7 +349,7 @@ struct GenerateDocumentation: AsyncSwiftCommand {
         try? swiftCommandState.fileSystem.removeFileTree(outputPath) // docc merge requires an empty output directory
         try swiftCommandState.fileSystem.createDirectory(outputPath, recursive: true)
 
-        print("MERGE")
+        print("MERGE: \(doccArchives)")
 
         let process = try Process.run(URL(fileURLWithPath: doccExecutable.pathString), arguments: [
                     "merge",
@@ -390,7 +363,5 @@ struct GenerateDocumentation: AsyncSwiftCommand {
         // TODO provide an option to set up an http server
         print("python3 -m http.server --directory \(outputPath)")
         print("http://localhost:8000/documentation")
-
-        // TODO figure out how to monitor for changes in preview mode so that it automatically updates itself (perhaps sourcekit-lsp/vscode is a much better way forward)
-    }
+      }
 }
