Merge pull request #1036 from allevato/separate-module-response-files

Refactor job response file handling to (hopefully) prevent regressions.
swiftlang · Mar 22, 2022 · 2ed97d4 · 2ed97d4
2 parents f84b001 + 7cf36a7
commit 2ed97d4
Show file tree

Hide file tree

Showing 31 changed files with 212 additions and 97 deletions.
diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift
@@ -172,7 +172,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
       jobs.append(Job(
         moduleName: moduleId.moduleName,
         kind: .emitModule,
-        tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+        tool: try toolchain.resolvedTool(.swiftCompiler),
         commandLine: commandLine,
         inputs: inputs,
         primaryInputs: [],
@@ -237,7 +237,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
         jobs.append(Job(
           moduleName: moduleId.moduleName,
           kind: .generatePCM,
-          tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+          tool: try toolchain.resolvedTool(.swiftCompiler),
           commandLine: commandLine,
           inputs: inputs,
           primaryInputs: [],

diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift
@@ -30,13 +30,12 @@ public extension Driver {
     // Construct the scanning job.
     return Job(moduleName: moduleOutputInfo.name,
                kind: .scanDependencies,
-               tool: VirtualPath.absolute(try toolchain.getToolPath(.swiftCompiler)),
+               tool: try toolchain.resolvedTool(.swiftCompiler),
                commandLine: commandLine,
                displayInputs: inputs,
                inputs: inputs,
                primaryInputs: [],
-               outputs: [TypedVirtualPath(file: .standardOutput, type: .jsonDependencies)],
-               supportsResponseFiles: false)
+               outputs: [TypedVirtualPath(file: .standardOutput, type: .jsonDependencies)])
   }
 
   /// Generate a full command-line invocation to be used for the dependency scanning action
@@ -263,13 +262,12 @@ public extension Driver {
     // Construct the scanning job.
     return Job(moduleName: moduleOutputInfo.name,
                kind: .scanDependencies,
-               tool: VirtualPath.absolute(try toolchain.getToolPath(.swiftCompiler)),
+               tool: try toolchain.resolvedTool(.swiftCompiler),
                commandLine: commandLine,
                displayInputs: inputs,
                inputs: inputs,
                primaryInputs: [],
-               outputs: [TypedVirtualPath(file: .standardOutput, type: .jsonDependencies)],
-               supportsResponseFiles: false)
+               outputs: [TypedVirtualPath(file: .standardOutput, type: .jsonDependencies)])
   }
 
   /// Precompute the dependencies for a given collection of modules using swift frontend's batch scanning mode
@@ -312,13 +310,12 @@ public extension Driver {
     // Construct the scanning job.
     return Job(moduleName: moduleOutputInfo.name,
                kind: .scanDependencies,
-               tool: VirtualPath.absolute(try toolchain.getToolPath(.swiftCompiler)),
+               tool: try toolchain.resolvedTool(.swiftCompiler),
                commandLine: commandLine,
                displayInputs: inputs,
                inputs: inputs,
                primaryInputs: [],
-               outputs: outputs,
-               supportsResponseFiles: false)
+               outputs: outputs)
   }
 
   /// Serialize a collection of modules into an input format expected by the batch module dependency scanner.

diff --git a/Sources/SwiftDriver/Jobs/APIDigesterJobs.swift b/Sources/SwiftDriver/Jobs/APIDigesterJobs.swift
@@ -56,12 +56,11 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: mode.baselineGenerationJobKind,
-      tool: .absolute(try toolchain.getToolPath(.swiftAPIDigester)),
+      tool: try toolchain.resolvedTool(.swiftAPIDigester),
       commandLine: commandLine,
       inputs: [.init(file: modulePath, type: .swiftModule)],
       primaryInputs: [],
-      outputs: [.init(file: outputPath, type: mode.baselineFileType)],
-      supportsResponseFiles: true
+      outputs: [.init(file: outputPath, type: mode.baselineFileType)]
     )
   }
 
@@ -100,12 +99,11 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .compareABIBaseline,
-      tool: .absolute(try toolchain.getToolPath(.swiftAPIDigester)),
+      tool: try toolchain.resolvedTool(.swiftAPIDigester),
       commandLine: commandLine,
       inputs: inputs,
       primaryInputs: [],
-      outputs: [diag],
-      supportsResponseFiles: true
+      outputs: [diag]
     )
   }
 
@@ -141,12 +139,11 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: mode.baselineComparisonJobKind,
-      tool: .absolute(try toolchain.getToolPath(.swiftAPIDigester)),
+      tool: try toolchain.resolvedTool(.swiftAPIDigester),
       commandLine: commandLine,
       inputs: inputs,
       primaryInputs: [],
-      outputs: [.init(file: serializedDiagnosticsPath ?? VirtualPath.Handle.standardOutput, type: .diagnostics)],
-      supportsResponseFiles: true
+      outputs: [.init(file: serializedDiagnosticsPath ?? VirtualPath.Handle.standardOutput, type: .diagnostics)]
     )
   }
 

diff --git a/Sources/SwiftDriver/Jobs/AutolinkExtractJob.swift b/Sources/SwiftDriver/Jobs/AutolinkExtractJob.swift
@@ -43,12 +43,11 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .autolinkExtract,
-      tool: .absolute(try toolchain.getToolPath(.swiftAutolinkExtract)),
+      tool: try toolchain.resolvedTool(.swiftAutolinkExtract),
       commandLine: commandLine,
       inputs: inputs,
       primaryInputs: [],
-      outputs: [.init(file: output.intern(), type: .autolink)],
-      supportsResponseFiles: true
+      outputs: [.init(file: output.intern(), type: .autolink)]
     )
   }
 }
diff --git a/Sources/SwiftDriver/Jobs/BackendJob.swift b/Sources/SwiftDriver/Jobs/BackendJob.swift
@@ -70,13 +70,12 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .backend,
-      tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+      tool: try toolchain.resolvedTool(.swiftCompiler),
       commandLine: commandLine,
       displayInputs: inputs,
       inputs: inputs,
       primaryInputs: [],
-      outputs: outputs,
-      supportsResponseFiles: true
+      outputs: outputs
     )
   }
 }
diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift
@@ -383,14 +383,13 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .compile,
-      tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+      tool: try toolchain.resolvedTool(.swiftCompiler),
       commandLine: commandLine,
       displayInputs: displayInputs,
       inputs: inputs,
       primaryInputs: primaryInputs,
       outputs: outputs,
-      inputOutputMap: inputOutputMap,
-      supportsResponseFiles: true
+      inputOutputMap: inputOutputMap
     )
   }
 }

diff --git a/Sources/SwiftDriver/Jobs/DarwinToolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/DarwinToolchain+LinkerSupport.swift
@@ -206,7 +206,7 @@ extension DarwinToolchain {
     lto: LTOKind?,
     sanitizers: Set<Sanitizer>,
     targetInfo: FrontendTargetInfo
-  ) throws -> AbsolutePath {
+  ) throws -> ResolvedTool {
     // Set up for linking.
     let linkerTool: Tool
     switch linkerOutputType {
@@ -251,7 +251,7 @@ extension DarwinToolchain {
     commandLine.appendFlag("-o")
     commandLine.appendPath(outputFile)
 
-    return try getToolPath(linkerTool)
+    return try resolvedTool(linkerTool)
   }
 
   private func addLinkInputs(shouldUseInputFileList: Bool,

diff --git a/Sources/SwiftDriver/Jobs/EmitModuleJob.swift b/Sources/SwiftDriver/Jobs/EmitModuleJob.swift
@@ -120,7 +120,7 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .emitModule,
-      tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+      tool: try toolchain.resolvedTool(.swiftCompiler),
       commandLine: commandLine,
       inputs: inputs,
       primaryInputs: [],

diff --git a/Sources/SwiftDriver/Jobs/EmitSupportedFeaturesJob.swift b/Sources/SwiftDriver/Jobs/EmitSupportedFeaturesJob.swift
@@ -40,14 +40,13 @@ extension Toolchain {
     return Job(
       moduleName: "",
       kind: .emitSupportedFeatures,
-      tool: .absolute(try getToolPath(.swiftCompiler)),
+      tool: try resolvedTool(.swiftCompiler),
       commandLine: commandLine,
       displayInputs: [],
       inputs: inputs,
       primaryInputs: [],
       outputs: [.init(file: .standardOutput, type: .jsonCompilerFeatures)],
-      requiresInPlaceExecution: requiresInPlaceExecution,
-      supportsResponseFiles: false
+      requiresInPlaceExecution: requiresInPlaceExecution
     )
   }
 }

diff --git a/Sources/SwiftDriver/Jobs/GenerateDSYMJob.swift b/Sources/SwiftDriver/Jobs/GenerateDSYMJob.swift
@@ -33,7 +33,7 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .generateDSYM,
-      tool: .absolute(try toolchain.getToolPath(.dsymutil)),
+      tool: try toolchain.resolvedTool(.dsymutil),
       commandLine: commandLine,
       displayInputs: [],
       inputs: inputs,

diff --git a/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift b/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift
@@ -64,13 +64,12 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .generatePCH,
-      tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+      tool: try toolchain.resolvedTool(.swiftCompiler),
       commandLine: commandLine,
       displayInputs: [],
       inputs: inputs,
       primaryInputs: [],
-      outputs: outputs,
-      supportsResponseFiles: true
+      outputs: outputs
     )
   }
 }
diff --git a/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift b/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift
@@ -55,7 +55,7 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .generatePCM,
-      tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+      tool: try toolchain.resolvedTool(.swiftCompiler),
       commandLine: commandLine,
       displayInputs: [],
       inputs: inputs,
@@ -84,7 +84,7 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .dumpPCM,
-      tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+      tool: try toolchain.resolvedTool(.swiftCompiler),
       commandLine: commandLine,
       displayInputs: [],
       inputs: inputs,

diff --git a/Sources/SwiftDriver/Jobs/GenericUnixToolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/GenericUnixToolchain+LinkerSupport.swift
@@ -58,7 +58,7 @@ extension GenericUnixToolchain {
     lto: LTOKind?,
     sanitizers: Set<Sanitizer>,
     targetInfo: FrontendTargetInfo
-  ) throws -> AbsolutePath {
+  ) throws -> ResolvedTool {
     let targetTriple = targetInfo.target.triple
     switch linkerOutputType {
     case .dynamicLibrary:
@@ -103,9 +103,9 @@ extension GenericUnixToolchain {
       // Windows rather than msvcprt).  When C++ interop is enabled, we will need to
       // surface this via a driver flag.  For now, opt for the simpler approach of
       // just using `clang` and avoid a dependency on the C++ runtime.
-      var clangPath = try parsedOptions.hasArgument(.enableExperimentalCxxInterop)
-                          ? getToolPath(.clangxx)
-                          : getToolPath(.clang)
+      let clangTool: Tool =
+        parsedOptions.hasArgument(.enableExperimentalCxxInterop) ? .clangxx : .clang
+      var clangPath = try getToolPath(clangTool)
       if let toolsDirPath = parsedOptions.getLastArgument(.toolsDirectory) {
         // FIXME: What if this isn't an absolute path?
         let toolsDir = try AbsolutePath(validating: toolsDirPath.asSingle)
@@ -296,7 +296,7 @@ extension GenericUnixToolchain {
       // This should be the last option, for convenience in checking output.
       commandLine.appendFlag(.o)
       commandLine.appendPath(outputFile)
-      return clangPath
+      return try resolvedTool(clangTool, pathOverride: clangPath)
     case .staticLibrary:
       // We're using 'ar' as a linker
       commandLine.appendFlag("crs")
@@ -308,9 +308,9 @@ extension GenericUnixToolchain {
                          }.map { .path($0.file) })
       if targetTriple.environment == .android {
         // Always use the LTO archiver llvm-ar for Android
-        return try getToolPath(.staticLinker(.llvmFull))
+        return try resolvedTool(.staticLinker(.llvmFull))
       } else {
-        return try getToolPath(.staticLinker(lto))
+        return try resolvedTool(.staticLinker(lto))
       }
     }
 

diff --git a/Sources/SwiftDriver/Jobs/InterpretJob.swift b/Sources/SwiftDriver/Jobs/InterpretJob.swift
@@ -44,14 +44,13 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .interpret,
-      tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+      tool: try toolchain.resolvedTool(.swiftCompiler),
       commandLine: commandLine,
       inputs: inputs,
       primaryInputs: [],
       outputs: [],
       extraEnvironment: extraEnvironment,
-      requiresInPlaceExecution: true,
-      supportsResponseFiles: true
+      requiresInPlaceExecution: true
     )
   }
 }
diff --git a/Sources/SwiftDriver/Jobs/Job.swift b/Sources/SwiftDriver/Jobs/Job.swift
@@ -106,20 +106,19 @@ public struct Job: Codable, Equatable, Hashable {
   public init(
     moduleName: String,
     kind: Kind,
-    tool: VirtualPath,
+    tool: ResolvedTool,
     commandLine: [ArgTemplate],
     displayInputs: [TypedVirtualPath]? = nil,
     inputs: [TypedVirtualPath],
     primaryInputs: [TypedVirtualPath],
     outputs: [TypedVirtualPath],
     inputOutputMap: [TypedVirtualPath : [TypedVirtualPath]] = [:],
     extraEnvironment: [String: String] = [:],
-    requiresInPlaceExecution: Bool = false,
-    supportsResponseFiles: Bool = false
+    requiresInPlaceExecution: Bool = false
   ) {
     self.moduleName = moduleName
     self.kind = kind
-    self.tool = tool
+    self.tool = .absolute(tool.path)
     self.commandLine = commandLine
     self.displayInputs = displayInputs ?? []
     self.inputs = inputs
@@ -128,7 +127,7 @@ public struct Job: Codable, Equatable, Hashable {
     self.compileInputOutputMap = inputOutputMap
     self.extraEnvironment = extraEnvironment
     self.requiresInPlaceExecution = requiresInPlaceExecution
-    self.supportsResponseFiles = supportsResponseFiles
+    self.supportsResponseFiles = tool.supportsResponseFiles
   }
 
   public var primarySwiftSourceFiles: [SwiftSourceFile] { primaryInputs.swiftSourceFiles }

diff --git a/Sources/SwiftDriver/Jobs/LinkJob.swift b/Sources/SwiftDriver/Jobs/LinkJob.swift
@@ -52,7 +52,7 @@ extension Driver {
     }
 
     // Defer to the toolchain for platform-specific linking
-    let toolPath = try toolchain.addPlatformSpecificLinkerArgs(
+    let linkTool = try toolchain.addPlatformSpecificLinkerArgs(
       to: &commandLine,
       parsedOptions: &parsedOptions,
       linkerOutputType: linkerOutputType!,
@@ -67,15 +67,12 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .link,
-      tool: .absolute(toolPath),
+      tool: linkTool,
       commandLine: commandLine,
       displayInputs: inputs,
       inputs: inputs,
       primaryInputs: [],
-      outputs: [.init(file: outputFile.intern(), type: .image)],
-      // FIXME: newer ld64 supports response files as well, though really,
-      // Darwin should use clang as the linker driver like the other targets
-      supportsResponseFiles: !frontendTargetInfo.target.triple.isDarwin
+      outputs: [.init(file: outputFile.intern(), type: .image)]
     )
   }
 }
diff --git a/Sources/SwiftDriver/Jobs/MergeModuleJob.swift b/Sources/SwiftDriver/Jobs/MergeModuleJob.swift
@@ -86,12 +86,11 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .mergeModule,
-      tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+      tool: try toolchain.resolvedTool(.swiftCompiler),
       commandLine: commandLine,
       inputs: inputs,
       primaryInputs: [],
-      outputs: outputs,
-      supportsResponseFiles: true
+      outputs: outputs
     )
   }
 }
diff --git a/Sources/SwiftDriver/Jobs/ModuleWrapJob.swift b/Sources/SwiftDriver/Jobs/ModuleWrapJob.swift
@@ -29,12 +29,11 @@ extension Driver {
     return Job(
       moduleName: moduleOutputInfo.name,
       kind: .moduleWrap,
-      tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+      tool: try toolchain.resolvedTool(.swiftCompiler),
       commandLine: commandLine,
       inputs: [moduleInput],
       primaryInputs: [],
-      outputs: [.init(file: outputPath.intern(), type: .object)],
-      supportsResponseFiles: true
+      outputs: [.init(file: outputPath.intern(), type: .object)]
     )
   }
 }