Merge pull request #2293 from ahoppen/multi-file-indexing

Support indexing of mulitple Swift files within the same compiler invocation

Author: ahoppen

Contributor Documentation/BSP Extensions.md

@@ -130,8 +130,12 @@ export interface SourceKitSourceItemData {
   kind?: "source" | "header" | "doccCatalog";
 
   /**
-   * The output path that is during indexing for this file, ie. the `-index-unit-output-path`, if it is specified
-   * in the compiler arguments or the file that is passed as `-o`, if `-index-unit-output-path` is not specified.
+   * The output path that is a string that uniquely identifies the index output of this file in this target. If an index
+   * store should be re-used between build and background indexing, it must match the `-o` path or
+   * `-index-unit-output-path` used during the build.
+   *
+   * The index unit output path historically matched the path used for `-o` during compilation but has since evolved to
+   * be an opaque string. In particular, it does not have to match to any file on disk.
    *
    * This allows SourceKit-LSP to remove index entries for source files that are removed from a target but remain
    * present on disk and to index a file that is part of multiple targets in the context of each target.

Sources/BuildServerIntegration/BuildSettingsLogger.swift

@@ -12,6 +12,7 @@
 
 package import LanguageServerProtocol
 package import SKLogging
+import SwiftExtensions
 
 // MARK: - Build settings logger
 
@@ -27,11 +28,31 @@ package actor BuildSettingsLogger {
     Self.log(level: level, settings: settings, for: uri)
   }
 
-  /// Log the given build settings.
+  /// Log the given build settings for a single file
   ///
   /// In contrast to the instance method `log`, this will always log the build settings. The instance method only logs
   /// the build settings if they have changed.
   package static func log(level: LogLevel = .default, settings: FileBuildSettings, for uri: DocumentURI) {
+    log(level: level, settings: settings, for: [uri])
+  }
+
+  /// Log the given build settings for a list of source files that all share the same build settings.
+  ///
+  /// In contrast to the instance method `log`, this will always log the build settings. The instance method only logs
+  /// the build settings if they have changed.
+  package static func log(level: LogLevel = .default, settings: FileBuildSettings, for uris: [DocumentURI]) {
+    let header: String
+    if let uri = uris.only {
+      header = "Build settings for \(uri.forLogging)"
+    } else if let firstUri = uris.first {
+      header = "Build settings for \(firstUri.forLogging) and \(firstUri) and \(uris.count - 1) others"
+    } else {
+      header = "Build settings for empty list"
+    }
+    log(level: level, settings: settings, header: header)
+  }
+
+  private static func log(level: LogLevel = .default, settings: FileBuildSettings, header: String) {
     let log = """
       Compiler Arguments:
       \(settings.compilerArguments.joined(separator: "\n"))
@@ -47,7 +68,7 @@ package actor BuildSettingsLogger {
       logger.log(
         level: level,
         """
-        Build settings for \(uri.forLogging) (\(index + 1)/\(chunks.count))
+        \(header) (\(index + 1)/\(chunks.count))
         \(chunk)
         """
       )

Sources/BuildServerIntegration/FileBuildSettings.swift

@@ -18,8 +18,7 @@ import LanguageServerProtocolExtensions
 ///
 /// Encapsulates all the settings needed to compile a single file, including the compiler arguments
 /// and working directory. `FileBuildSettings`` are typically the result of a build server query.
-package struct FileBuildSettings: Equatable, Sendable {
-
+package struct FileBuildSettings: Hashable, Sendable {
   /// The compiler arguments to use for this file.
   package var compilerArguments: [String]
 

Sources/BuildServerIntegration/SwiftPMBuildServer.swift

@@ -514,6 +514,11 @@ package actor SwiftPMBuildServer: BuiltInBuildServer {
 
       throw NonFileURIError(uri: file)
     }
+    #if compiler(>=6.4)
+    #warning(
+      "Once we can guarantee that the toolchain can index multiple Swift files in a single invocation, we no longer need to set -index-unit-output-path since it's always set using an -output-file-map"
+    )
+    #endif
     var compilerArguments = try buildTarget.compileArguments(for: fileURL)
     if buildTarget.compiler == .swift {
       compilerArguments += [

Sources/SKTestSupport/SkipUnless.swift

@@ -502,6 +502,15 @@ package actor SkipUnless {
     }
   }
 
+  package static func canIndexMultipleSwiftFilesInSingleInvocation(
+    file: StaticString = #filePath,
+    line: UInt = #line
+  ) async throws {
+    return try await shared.skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(6, 3), file: file, line: line) {
+      return await ToolchainRegistry.forTesting.default?.canIndexMultipleSwiftFilesInSingleInvocation ?? false
+    }
+  }
+
   private static func getSourceKitD() async throws -> SourceKitD {
     guard let sourcekitdPath = await ToolchainRegistry.forTesting.default?.sourcekitd else {
       throw GenericError("Could not find SourceKitD")

Sources/SemanticIndex/SemanticIndexManager.swift

@@ -921,7 +921,13 @@ package final actor SemanticIndexManager {
     // (https://github.com/swiftlang/sourcekit-lsp/issues/1262)
     for targetsBatch in sortedTargets.partition(intoBatchesOfSize: 1) {
       let preparationTaskID = UUID()
-      let filesToIndex = targetsBatch.flatMap({ filesByTarget[$0]! })
+      let filesToIndex = targetsBatch.flatMap { (target) -> [FileIndexInfo] in
+        guard let files = filesByTarget[target] else {
+          logger.fault("Unexpectedly found no files for target in target batch")
+          return []
+        }
+        return files
+      }
 
       // First schedule preparation of the targets. We schedule the preparation outside of `indexTask` so that we
       // deterministically prepare targets in the topological order for indexing. If we triggered preparation inside the
@@ -950,36 +956,36 @@ package final actor SemanticIndexManager {
 
         // And after preparation is done, index the files in the targets.
         await withTaskGroup(of: Void.self) { taskGroup in
-          for target in targetsBatch {
-            var filesByLanguage: [Language: [FileIndexInfo]] = [:]
-            for fileInfo in filesByTarget[target]! {
-              filesByLanguage[fileInfo.language, default: []].append(fileInfo)
+          let fileInfos = targetsBatch.flatMap { (target) -> [FileIndexInfo] in
+            guard let files = filesByTarget[target] else {
+              logger.fault("Unexpectedly found no files for target in target batch")
+              return []
             }
-            for (language, fileInfos) in filesByLanguage {
-              // TODO: Once swiftc supports indexing of multiple files in a single invocation, increase the batch size to
-              // allow it to share AST builds between multiple files within a target.
-              // (https://github.com/swiftlang/sourcekit-lsp/issues/1268)
-              for fileBatch in fileInfos.partition(intoBatchesOfSize: 1) {
-                taskGroup.addTask {
-                  let fileAndOutputPaths: [FileAndOutputPath] = fileBatch.compactMap {
-                    guard $0.target == target else {
-                      logger.fault(
-                        "FileIndexInfo refers to different target than should be indexed \($0.target.forLogging) vs \(target.forLogging)"
-                      )
-                      return nil
-                    }
-                    return FileAndOutputPath(file: $0.file, outputPath: $0.outputPath)
-                  }
-                  await self.updateIndexStore(
-                    for: fileAndOutputPaths,
-                    target: target,
-                    language: language,
-                    indexFilesWithUpToDateUnit: indexFilesWithUpToDateUnit,
-                    preparationTaskID: preparationTaskID,
-                    priority: priority
+            return files
+          }
+          let batches = await UpdateIndexStoreTaskDescription.batches(
+            toIndex: fileInfos,
+            buildServerManager: buildServerManager
+          )
+          for (target, language, fileBatch) in batches {
+            taskGroup.addTask {
+              let fileAndOutputPaths: [FileAndOutputPath] = fileBatch.compactMap {
+                guard $0.target == target else {
+                  logger.fault(
+                    "FileIndexInfo refers to different target than should be indexed: \($0.target.forLogging) vs \(target.forLogging)"
                   )
+                  return nil
                 }
+                return FileAndOutputPath(file: $0.file, outputPath: $0.outputPath)
               }
+              await self.updateIndexStore(
+                for: fileAndOutputPaths,
+                target: target,
+                language: language,
+                indexFilesWithUpToDateUnit: indexFilesWithUpToDateUnit,
+                preparationTaskID: preparationTaskID,
+                priority: priority
+              )
             }
           }
           await taskGroup.waitForAll()