[Caching] Path remapping support

Teach swift-driver to remap path when needed. This involves:
* Infer SDK/toolchain related path that needs to remapped
* Send path remap related arguments to swift-frontend for depscanning
* Send reverse map to swift-frontend for diagnostics replay
* Constructing the compilation command for main module using remapped
  path

Author: cachemeifyoucan

Sources/SwiftDriver/Driver/Driver.swift

@@ -224,8 +224,10 @@ public struct Driver {
   /// Should use file lists for inputs (number of inputs exceeds `fileListThreshold`).
   let shouldUseInputFileList: Bool
 
-  /// VirtualPath for shared all sources file list. `nil` if unused.
-  @_spi(Testing) public let allSourcesFileList: VirtualPath?
+  /// VirtualPath for shared all sources file list. `nil` if unused. This is used as a cache for
+  /// the file list computed during CompileJob creation and only holds valid to be query by tests
+  /// after planning to build.
+  @_spi(Testing) public var allSourcesFileList: VirtualPath? = nil
 
   /// The mode in which the compiler will execute.
   @_spi(Testing) public let compilerMode: CompilerMode
@@ -272,6 +274,35 @@ public struct Driver {
   let enableCaching: Bool
   let useClangIncludeTree: Bool
 
+  /// Scanner prefix mapping.
+  let scannerPrefixMap: [AbsolutePath: AbsolutePath]
+  let scannerPrefixMapSDK: AbsolutePath?
+  let scannerPrefixMapToolchain: AbsolutePath?
+  lazy var prefixMapping: [(AbsolutePath, AbsolutePath)] = {
+    var mapping: [(AbsolutePath, AbsolutePath)] = scannerPrefixMap.map {
+      return ($0.key, $0.value)
+    }
+    do {
+      guard isFrontendArgSupported(.scannerPrefixMap) else {
+        return []
+      }
+      if let sdkMapping = scannerPrefixMapSDK,
+         let sdkPath = absoluteSDKPath {
+        mapping.append((sdkPath, sdkMapping))
+      }
+      if let toolchainMapping = scannerPrefixMapToolchain {
+        let toolchainPath = try toolchain.executableDir.parentDirectory // usr
+                                                       .parentDirectory // toolchain
+        mapping.append((toolchainPath, toolchainMapping))
+      }
+      // The mapping needs to be sorted so the mapping is determinisitic.
+      // The sorting order is reversed so /tmp/tmp is preferred over /tmp in remapping.
+      return mapping.sorted { $0.0 > $1.0 }
+    } catch {
+      return mapping.sorted { $0.0 > $1.0 }
+    }
+  }()
+
   /// Code & data for incremental compilation. Nil if not running in incremental mode.
   /// Set during planning because needs the jobs to look at outputs.
   @_spi(Testing) public private(set) var incrementalCompilationState: IncrementalCompilationState? = nil
@@ -598,6 +629,17 @@ public struct Driver {
     let cachingEnableOverride = parsedOptions.hasArgument(.driverExplicitModuleBuild) && env.keys.contains("SWIFT_ENABLE_CACHING")
     self.enableCaching = parsedOptions.hasArgument(.cacheCompileJob) || cachingEnableOverride
     self.useClangIncludeTree = enableCaching && env.keys.contains("SWIFT_CACHING_USE_INCLUDE_TREE")
+    self.scannerPrefixMap = try Self.computeScanningPrefixMapper(&parsedOptions)
+    if let sdkMapping =  parsedOptions.getLastArgument(.scannerPrefixMapSdk)?.asSingle {
+      self.scannerPrefixMapSDK = try AbsolutePath(validating: sdkMapping)
+    } else {
+      self.scannerPrefixMapSDK = nil
+    }
+    if let toolchainMapping = parsedOptions.getLastArgument(.scannerPrefixMapToolchain)?.asSingle {
+      self.scannerPrefixMapToolchain = try AbsolutePath(validating: toolchainMapping)
+    } else {
+      self.scannerPrefixMapToolchain = nil
+    }
 
     // Compute the working directory.
     workingDirectory = try parsedOptions.getLastArgument(.workingDirectory).map { workingDirectoryArg in
@@ -678,13 +720,6 @@ public struct Driver {
 
     self.fileListThreshold = try Self.computeFileListThreshold(&self.parsedOptions, diagnosticsEngine: diagnosticsEngine)
     self.shouldUseInputFileList = inputFiles.count > fileListThreshold
-    if shouldUseInputFileList {
-      let swiftInputs = inputFiles.filter(\.type.isPartOfSwiftCompilation)
-      self.allSourcesFileList = try VirtualPath.createUniqueFilelist(RelativePath(validating: "sources"),
-                                                                     .list(swiftInputs.map(\.file)))
-    } else {
-      self.allSourcesFileList = nil
-    }
 
     self.lto = Self.ltoKind(&parsedOptions, diagnosticsEngine: diagnosticsEngine)
     // Figure out the primary outputs from the driver.
@@ -3524,4 +3559,18 @@ extension Driver {
     }
     return options
   }
+
+  static func computeScanningPrefixMapper(_ parsedOptions: inout ParsedOptions) throws -> [AbsolutePath: AbsolutePath] {
+    var mapping: [AbsolutePath: AbsolutePath] = [:]
+    for opt in parsedOptions.arguments(for: .scannerPrefixMap) {
+      let pluginArg = opt.argument.asSingle.split(separator: "=", maxSplits: 1)
+      if pluginArg.count != 2 {
+        throw Error.invalidArgumentValue(Option.scannerPrefixMap.spelling, opt.argument.asSingle)
+      }
+      let key = try AbsolutePath(validating: String(pluginArg[0]))
+      let value = try AbsolutePath(validating: String(pluginArg[1]))
+      mapping[key] = value
+    }
+    return mapping
+  }
 }

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

@@ -124,6 +124,14 @@ public extension Driver {
       try commandLine.appendLast(.clangScannerModuleCachePath, from: &parsedOptions)
     }
 
+    if isFrontendArgSupported(.scannerPrefixMap) {
+      // construct `-scanner-prefix-mapper` for scanner.
+      for (key, value) in prefixMapping {
+        commandLine.appendFlag(.scannerPrefixMap)
+        commandLine.appendFlag(key.pathString + "=" + value.pathString)
+      }
+    }
+
     // Pass on the input files
     commandLine.append(contentsOf: inputFiles.filter { $0.type == .swift }.map { .path($0.file) })
     return (inputs, commandLine)

Sources/SwiftDriver/Jobs/CompileJob.swift

@@ -112,21 +112,26 @@ extension Driver {
                                  outputType: FileType?,
                                  commandLine: inout [Job.ArgTemplate])
   throws -> ([TypedVirtualPath], [TypedVirtualPath]) {
-    let useInputFileList: Bool
-    if let allSourcesFileList = allSourcesFileList {
-      useInputFileList = true
+    let useInputFileList = shouldUseInputFileList
+    if let sourcesFileList = allSourcesFileList {
       commandLine.appendFlag(.filelist)
-      commandLine.appendPath(allSourcesFileList)
-    } else {
-      useInputFileList = false
+      commandLine.appendPath(sourcesFileList)
+    } else if shouldUseInputFileList {
+      let swiftInputs = inputFiles.filter(\.type.isPartOfSwiftCompilation)
+      let remappedSourcesFileList = try VirtualPath.createUniqueFilelist(RelativePath(validating: "sources"),
+                                                                         .list(swiftInputs.map{ return remapPath($0.file) }))
+      // Remember the filelist created.
+      self.allSourcesFileList = remappedSourcesFileList
+      commandLine.appendFlag(.filelist)
+      commandLine.appendPath(remappedSourcesFileList)
     }
 
     let usePrimaryInputFileList = primaryInputs.count > fileListThreshold
     if usePrimaryInputFileList {
       // primary file list
       commandLine.appendFlag(.primaryFilelist)
       let fileList = try VirtualPath.createUniqueFilelist(RelativePath(validating: "primaryInputs"),
-                                                          .list(primaryInputs.map(\.file)))
+                                                          .list(primaryInputs.map{ return remapPath($0.file) }))
       commandLine.appendPath(fileList)
     }
 
@@ -166,12 +171,11 @@ extension Driver {
       let isPrimary = usesPrimaryFileInputs && primaryInputFiles.contains(input)
       if isPrimary {
         if !usePrimaryInputFileList {
-          commandLine.appendFlag(.primaryFile)
-          commandLine.appendPath(input.file)
+          try addPathOption(option: .primaryFile, path: input.file, to:&commandLine)
         }
       } else {
         if !useInputFileList {
-          commandLine.appendPath(input.file)
+          try addPathArgument(input.file, to: &commandLine)
         }
       }