Refactor job response file handling to (hopefully) prevent regressions.

The new "emit module job" in Swift 5.6 was not marked as supporting response
files (even though it's a `swift-frontend` invocation, all of which should
support response files). This requires certain long `swiftc` invocations to
pass `-no-emit-module-separately` in order to succeed.

There are some other inconsistencies as well; for example, "emit PCM job"
as implemented today does not support response files, when it should.

This change refactors the interaction between `Tool`s and `Job`s. Whether
or not response files are accepted is a property of the **tool** (and the
toolchain), not of individual jobs. So, this adds a `supportsResponseFiles`
method to `Tool` that returns that value (with the toolchain as an argument
so that it can be conditional based on that, as it is for linking).

Then, instead of asking the toolchain for the path to a tool when creating
a job, you ask it for a `ResolvedTool`, which encapsulates the path and the
knowledge about response files. The path can still be overridden if needed
(as it is for linking in non-Darwin toolchains).

This change aims to prevent future regressions for response file handling
in a couple ways:

*   When creating a job, the caller no longer has to remember to pass a
    parameter indiciating whether it supports response files. (Worse,
    that parameter was optional, so auto-complete could easily leave
    it out!)
*   When introducing a new **tool,** the exhaustive switch in
    `Tool.supportsResponseFiles(in:)` requires the author of the change
    to acknowledge how it handles response files.
*   I've added regression tests to handle the common cases that involve
    large command lines (compiling Swift and generating PCMs).

Author: allevato

Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift

@@ -172,7 +172,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
       jobs.append(Job(
         moduleName: moduleId.moduleName,
         kind: .emitModule,
-        tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+        tool: try toolchain.resolvedTool(.swiftCompiler),
         commandLine: commandLine,
         inputs: inputs,
         primaryInputs: [],
@@ -237,7 +237,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
         jobs.append(Job(
           moduleName: moduleId.moduleName,
           kind: .generatePCM,
-          tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+          tool: try toolchain.resolvedTool(.swiftCompiler),
           commandLine: commandLine,
           inputs: inputs,
           primaryInputs: [],

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

@@ -30,13 +30,12 @@ public extension Driver {
     // Construct the scanning job.
     return Job(moduleName: moduleOutputInfo.name,
                kind: .scanDependencies,
-               tool: VirtualPath.absolute(try toolchain.getToolPath(.swiftCompiler)),
+               tool: try toolchain.resolvedTool(.swiftCompiler),
                commandLine: commandLine,
                displayInputs: inputs,
                inputs: inputs,
                primaryInputs: [],
-               outputs: [TypedVirtualPath(file: .standardOutput, type: .jsonDependencies)],
-               supportsResponseFiles: false)
+               outputs: [TypedVirtualPath(file: .standardOutput, type: .jsonDependencies)])
   }
 
   /// Generate a full command-line invocation to be used for the dependency scanning action
@@ -263,13 +262,12 @@ public extension Driver {
     // Construct the scanning job.
     return Job(moduleName: moduleOutputInfo.name,
                kind: .scanDependencies,
-               tool: VirtualPath.absolute(try toolchain.getToolPath(.swiftCompiler)),
+               tool: try toolchain.resolvedTool(.swiftCompiler),
                commandLine: commandLine,
                displayInputs: inputs,
                inputs: inputs,
                primaryInputs: [],
-               outputs: [TypedVirtualPath(file: .standardOutput, type: .jsonDependencies)],
-               supportsResponseFiles: false)
+               outputs: [TypedVirtualPath(file: .standardOutput, type: .jsonDependencies)])
   }
 
   /// Precompute the dependencies for a given collection of modules using swift frontend's batch scanning mode
@@ -312,13 +310,12 @@ public extension Driver {
     // Construct the scanning job.
     return Job(moduleName: moduleOutputInfo.name,
                kind: .scanDependencies,
-               tool: VirtualPath.absolute(try toolchain.getToolPath(.swiftCompiler)),
+               tool: try toolchain.resolvedTool(.swiftCompiler),
                commandLine: commandLine,
                displayInputs: inputs,
                inputs: inputs,
                primaryInputs: [],
-               outputs: outputs,
-               supportsResponseFiles: false)
+               outputs: outputs)
   }
 
   /// Serialize a collection of modules into an input format expected by the batch module dependency scanner.

Sources/SwiftDriver/Jobs/APIDigesterJobs.swift

@@ -56,12 +56,11 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: mode.baselineGenerationJobKind,
-      tool: .absolute(try toolchain.getToolPath(.swiftAPIDigester)),
+      tool: try toolchain.resolvedTool(.swiftAPIDigester),
       commandLine: commandLine,
       inputs: [.init(file: modulePath, type: .swiftModule)],
       primaryInputs: [],
-      outputs: [.init(file: outputPath, type: mode.baselineFileType)],
-      supportsResponseFiles: true
+      outputs: [.init(file: outputPath, type: mode.baselineFileType)]
     )
   }
 
@@ -100,12 +99,11 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .compareABIBaseline,
-      tool: .absolute(try toolchain.getToolPath(.swiftAPIDigester)),
+      tool: try toolchain.resolvedTool(.swiftAPIDigester),
       commandLine: commandLine,
       inputs: inputs,
       primaryInputs: [],
-      outputs: [diag],
-      supportsResponseFiles: true
+      outputs: [diag]
     )
   }
 
@@ -141,12 +139,11 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: mode.baselineComparisonJobKind,
-      tool: .absolute(try toolchain.getToolPath(.swiftAPIDigester)),
+      tool: try toolchain.resolvedTool(.swiftAPIDigester),
       commandLine: commandLine,
       inputs: inputs,
       primaryInputs: [],
-      outputs: [.init(file: serializedDiagnosticsPath ?? VirtualPath.Handle.standardOutput, type: .diagnostics)],
-      supportsResponseFiles: true
+      outputs: [.init(file: serializedDiagnosticsPath ?? VirtualPath.Handle.standardOutput, type: .diagnostics)]
     )
   }
 

Sources/SwiftDriver/Jobs/AutolinkExtractJob.swift

@@ -43,12 +43,11 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .autolinkExtract,
-      tool: .absolute(try toolchain.getToolPath(.swiftAutolinkExtract)),
+      tool: try toolchain.resolvedTool(.swiftAutolinkExtract),
       commandLine: commandLine,
       inputs: inputs,
       primaryInputs: [],
-      outputs: [.init(file: output.intern(), type: .autolink)],
-      supportsResponseFiles: true
+      outputs: [.init(file: output.intern(), type: .autolink)]
     )
   }
 }

Sources/SwiftDriver/Jobs/BackendJob.swift

@@ -70,13 +70,12 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .backend,
-      tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+      tool: try toolchain.resolvedTool(.swiftCompiler),
       commandLine: commandLine,
       displayInputs: inputs,
       inputs: inputs,
       primaryInputs: [],
-      outputs: outputs,
-      supportsResponseFiles: true
+      outputs: outputs
     )
   }
 }

Sources/SwiftDriver/Jobs/CompileJob.swift

@@ -383,14 +383,13 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .compile,
-      tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+      tool: try toolchain.resolvedTool(.swiftCompiler),
       commandLine: commandLine,
       displayInputs: displayInputs,
       inputs: inputs,
       primaryInputs: primaryInputs,
       outputs: outputs,
-      inputOutputMap: inputOutputMap,
-      supportsResponseFiles: true
+      inputOutputMap: inputOutputMap
     )
   }
 }

Sources/SwiftDriver/Jobs/DarwinToolchain+LinkerSupport.swift

@@ -206,7 +206,7 @@ extension DarwinToolchain {
     lto: LTOKind?,
     sanitizers: Set<Sanitizer>,
     targetInfo: FrontendTargetInfo
-  ) throws -> AbsolutePath {
+  ) throws -> ResolvedTool {
     // Set up for linking.
     let linkerTool: Tool
     switch linkerOutputType {
@@ -251,7 +251,7 @@ extension DarwinToolchain {
     commandLine.appendFlag("-o")
     commandLine.appendPath(outputFile)
 
-    return try getToolPath(linkerTool)
+    return try resolvedTool(linkerTool)
   }
 
   private func addLinkInputs(shouldUseInputFileList: Bool,

Sources/SwiftDriver/Jobs/EmitModuleJob.swift

@@ -120,7 +120,7 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .emitModule,
-      tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+      tool: try toolchain.resolvedTool(.swiftCompiler),
       commandLine: commandLine,
       inputs: inputs,
       primaryInputs: [],

Sources/SwiftDriver/Jobs/EmitSupportedFeaturesJob.swift

@@ -40,14 +40,13 @@ extension Toolchain {
     return Job(
       moduleName: "",
       kind: .emitSupportedFeatures,
-      tool: .absolute(try getToolPath(.swiftCompiler)),
+      tool: try resolvedTool(.swiftCompiler),
       commandLine: commandLine,
       displayInputs: [],
       inputs: inputs,
       primaryInputs: [],
       outputs: [.init(file: .standardOutput, type: .jsonCompilerFeatures)],
-      requiresInPlaceExecution: requiresInPlaceExecution,
-      supportsResponseFiles: false
+      requiresInPlaceExecution: requiresInPlaceExecution
     )
   }
 }

Sources/SwiftDriver/Jobs/GenerateDSYMJob.swift

@@ -33,7 +33,7 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .generateDSYM,
-      tool: .absolute(try toolchain.getToolPath(.dsymutil)),
+      tool: try toolchain.resolvedTool(.dsymutil),
       commandLine: commandLine,
       displayInputs: [],
       inputs: inputs,