Add -prefix-serialized-debugging-options (#39555)

This commit adds a new frontend flag that applies debug path prefixing to the
paths serialized in swiftmodule files. This makes it possible to use swiftmodule
files that have been built on different machines by applying the inverse map
when debugging, in a similar fashion to source path prefixing.

The inverse mapping in LLDB will be handled in a follow up PR.

Second pass at #39138

Tests updated to handle windows path separators.

This reverts commit f5aa95b.

Author: rmaz

include/swift/AST/SearchPathOptions.h

@@ -14,6 +14,7 @@
 #define SWIFT_AST_SEARCHPATHOPTIONS_H
 
 #include "swift/Basic/ArrayRefView.h"
+#include "swift/Basic/PathRemapper.h"
 #include "llvm/ADT/Hashing.h"
 
 #include <string>
@@ -97,6 +98,11 @@ class SearchPathOptions {
 
   /// A file containing modules we should perform batch scanning.
   std::string BatchScanInputFilePath;
+
+  /// Debug path mappings to apply to serialized search paths. These are
+  /// specified in LLDB from the target.source-map entries.
+  PathRemapper SearchPathRemapper;
+
 private:
   static StringRef
   pathStringFromFrameworkSearchPath(const FrameworkSearchPath &next) {

include/swift/Basic/LangOptions.h

@@ -772,6 +772,9 @@ namespace swift {
                           DisableOverlayModules,
                           EnableClangSPI);
     }
+
+    std::vector<std::string> getRemappedExtraArgs(
+        std::function<std::string(StringRef)> pathRemapCallback) const;
   };
 
 } // end namespace swift

include/swift/Frontend/FrontendOptions.h

@@ -182,6 +182,11 @@ class FrontendOptions {
   /// module appears to not be a public module.
   Optional<bool> SerializeOptionsForDebugging;
 
+  /// When true the debug prefix map entries will be applied to debugging
+  /// options before serialization. These can be reconstructed at debug time by
+  /// applying the inverse map in SearchPathOptions.SearchPathRemapper.
+  bool DebugPrefixSerializedDebuggingOptions = false;
+
   /// When true, check if all required SwiftOnoneSupport symbols are present in
   /// the module.
   bool CheckOnoneSupportCompleteness = false;

include/swift/Option/Options.td

@@ -559,7 +559,7 @@ def disable_bridging_pch : Flag<["-"], "disable-bridging-pch">,
 def lto : Joined<["-"], "lto=">,
   Flags<[FrontendOption, NoInteractiveOption]>,
   HelpText<"Specify the LTO type to either 'llvm-thin' or 'llvm-full'">;
-  
+
 def lto_library : Separate<["-"], "lto-library">,
   Flags<[FrontendOption, ArgumentIsPath, NoInteractiveOption]>,
   HelpText<"Perform LTO with <lto-library>">, MetaVarName<"<lto-library>">;
@@ -815,6 +815,10 @@ def debug_info_format : Joined<["-"], "debug-info-format=">,
   Flags<[FrontendOption]>,
   HelpText<"Specify the debug info format type to either 'dwarf' or 'codeview'">;
 
+def prefix_serialized_debugging_options : Flag<["-"], "prefix-serialized-debugging-options">,
+  Flags<[FrontendOption]>,
+  HelpText<"Apply debug prefix mappings to serialized debug info in Swiftmodule files">;
+
 // Verify debug info
 def verify_debug_info : Flag<["-"], "verify-debug-info">,
   Flags<[NoInteractiveOption, DoesNotAffectIncrementalBuild]>,

include/swift/Serialization/SerializationOptions.h

@@ -14,6 +14,7 @@
 #define SWIFT_SERIALIZATION_SERIALIZATIONOPTIONS_H
 
 #include "swift/Basic/LLVM.h"
+#include "swift/Basic/PathRemapper.h"
 #include "llvm/Support/VersionTuple.h"
 
 namespace swift {
@@ -42,7 +43,10 @@ namespace swift {
     StringRef ImportedHeader;
     StringRef ModuleLinkName;
     StringRef ModuleInterface;
-    ArrayRef<std::string> ExtraClangOptions;
+    std::vector<std::string> ExtraClangOptions;
+
+    /// Path prefixes that should be rewritten in debug info.
+    PathRemapper DebuggingOptionsPrefixMap;
 
     /// Describes a single-file dependency for this module, along with the
     /// appropriate strategy for how to verify if it's up-to-date.

lib/Basic/LangOptions.cpp

@@ -408,3 +408,56 @@ DiagnosticBehavior LangOptions::getAccessNoteFailureLimit() const {
   }
   llvm_unreachable("covered switch");
 }
+
+std::vector<std::string> ClangImporterOptions::getRemappedExtraArgs(
+    std::function<std::string(StringRef)> pathRemapCallback) const {
+  auto consumeIncludeOption = [](StringRef &arg, StringRef &prefix) {
+    static StringRef options[] = {"-I",
+                                  "-F",
+                                  "-fmodule-map-file=",
+                                  "-iquote",
+                                  "-idirafter",
+                                  "-iframeworkwithsysroot",
+                                  "-iframework",
+                                  "-iprefix",
+                                  "-iwithprefixbefore",
+                                  "-iwithprefix",
+                                  "-isystemafter",
+                                  "-isystem",
+                                  "-isysroot",
+                                  "-ivfsoverlay",
+                                  "-working-directory=",
+                                  "-working-directory"};
+    for (StringRef &option : options)
+      if (arg.consume_front(option)) {
+        prefix = option;
+        return true;
+      }
+    return false;
+  };
+
+  // true if the previous argument was the dash-option of an option pair
+  bool remap_next = false;
+  std::vector<std::string> args;
+  for (auto A : ExtraArgs) {
+    StringRef prefix;
+    StringRef arg(A);
+
+    if (remap_next) {
+      remap_next = false;
+      args.push_back(pathRemapCallback(arg));
+    } else if (consumeIncludeOption(arg, prefix)) {
+      if (arg.empty()) {
+        // Option pair
+        remap_next = true;
+        args.push_back(prefix.str());
+      } else {
+        // Combine prefix with remapped path value
+        args.push_back(prefix.str() + pathRemapCallback(arg));
+      }
+    } else {
+      args.push_back(A);
+    }
+  }
+  return args;
+}

lib/Frontend/ArgsToFrontendOptionsConverter.cpp

@@ -258,6 +258,8 @@ bool ArgsToFrontendOptionsConverter::convert(
         A->getOption().matches(OPT_serialize_debugging_options);
   }
 
+  Opts.DebugPrefixSerializedDebuggingOptions |=
+      Args.hasArg(OPT_prefix_serialized_debugging_options);
   Opts.EnableSourceImport |= Args.hasArg(OPT_enable_source_import);
   Opts.ImportUnderlyingModule |= Args.hasArg(OPT_import_underlying_module);
   Opts.EnableIncrementalDependencyVerifier |= Args.hasArg(OPT_verify_incremental_dependencies);

lib/Frontend/Frontend.cpp

@@ -147,7 +147,7 @@ SerializationOptions CompilerInvocation::computeSerializationOptions(
     serializationOpts.ImportedHeader = opts.ImplicitObjCHeaderPath;
   serializationOpts.ModuleLinkName = opts.ModuleLinkName;
   serializationOpts.UserModuleVersion = opts.UserModuleVersion;
-  serializationOpts.ExtraClangOptions = getClangImporterOptions().ExtraArgs;
+
   serializationOpts.PublicDependentLibraries =
       getIRGenOptions().PublicLinkLibraries;
   serializationOpts.SDKName = getLangOptions().SDKName;
@@ -176,6 +176,20 @@ SerializationOptions CompilerInvocation::computeSerializationOptions(
       opts.SerializeOptionsForDebugging.getValueOr(
           !module->isExternallyConsumed());
 
+  if (serializationOpts.SerializeOptionsForDebugging &&
+      opts.DebugPrefixSerializedDebuggingOptions) {
+    serializationOpts.DebuggingOptionsPrefixMap =
+        getIRGenOptions().DebugPrefixMap;
+    auto &remapper = serializationOpts.DebuggingOptionsPrefixMap;
+    auto remapClangPaths = [&remapper](StringRef path) {
+      return remapper.remapPath(path);
+    };
+    serializationOpts.ExtraClangOptions =
+        getClangImporterOptions().getRemappedExtraArgs(remapClangPaths);
+  } else {
+    serializationOpts.ExtraClangOptions = getClangImporterOptions().ExtraArgs;
+  }
+
   serializationOpts.DisableCrossModuleIncrementalInfo =
       opts.DisableCrossModuleIncrementalBuild;
 

lib/Serialization/ModuleFile.cpp

@@ -158,9 +158,11 @@ Status ModuleFile::associateWithFileContext(FileUnit *file, SourceLoc diagLoc,
     return error(status);
   }
 
-  for (const auto &searchPath : Core->SearchPaths)
-    ctx.addSearchPath(searchPath.Path, searchPath.IsFramework,
-                      searchPath.IsSystem);
+  for (const auto &searchPath : Core->SearchPaths) {
+    ctx.addSearchPath(
+        ctx.SearchPathOpts.SearchPathRemapper.remapPath(searchPath.Path),
+        searchPath.IsFramework, searchPath.IsSystem);
+  }
 
   auto clangImporter = static_cast<ClangImporter *>(ctx.getClangModuleLoader());
 

lib/Serialization/Serialization.cpp

@@ -40,6 +40,7 @@
 #include "swift/Basic/Defer.h"
 #include "swift/Basic/Dwarf.h"
 #include "swift/Basic/FileSystem.h"
+#include "swift/Basic/PathRemapper.h"
 #include "swift/Basic/STLExtras.h"
 #include "swift/Basic/Version.h"
 #include "swift/ClangImporter/ClangImporter.h"
@@ -48,9 +49,9 @@
 #include "swift/Demangling/ManglingMacros.h"
 #include "swift/Serialization/SerializationOptions.h"
 #include "swift/Strings.h"
-#include "clang/AST/DeclTemplate.h"
-#include "swift/SymbolGraphGen/SymbolGraphOptions.h"
 #include "swift/SymbolGraphGen/SymbolGraphGen.h"
+#include "swift/SymbolGraphGen/SymbolGraphOptions.h"
+#include "clang/AST/DeclTemplate.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
@@ -511,7 +512,7 @@ static uint8_t getRawOpaqueReadOwnership(swift::OpaqueReadOwnership ownership) {
   CASE(OwnedOrBorrowed)
 #undef CASE
   }
-  llvm_unreachable("bad kind");  
+  llvm_unreachable("bad kind");
 }
 
 static uint8_t getRawReadImplKind(swift::ReadImplKind kind) {
@@ -1062,25 +1063,35 @@ void Serializer::writeHeader(const SerializationOptions &options) {
         options_block::SDKPathLayout SDKPath(Out);
         options_block::XCCLayout XCC(Out);
 
-        SDKPath.emit(ScratchRecord, M->getASTContext().SearchPathOpts.SDKPath);
+        const auto &PathRemapper = options.DebuggingOptionsPrefixMap;
+        SDKPath.emit(
+            ScratchRecord,
+            PathRemapper.remapPath(M->getASTContext().SearchPathOpts.SDKPath));
         auto &Opts = options.ExtraClangOptions;
-        for (auto Arg = Opts.begin(), E = Opts.end(); Arg != E; ++Arg) { 
-          // FIXME: This is a hack and calls for a better design.
-          //
-          // Filter out any -ivfsoverlay options that include an
-          // unextended-module-overlay.yaml overlay. By convention the Xcode
-          // buildsystem uses these while *building* mixed Objective-C and Swift
-          // frameworks; but they should never be used to *import* the module
-          // defined in the framework.
-          if (StringRef(*Arg).startswith("-ivfsoverlay")) {
+        for (auto Arg = Opts.begin(), E = Opts.end(); Arg != E; ++Arg) {
+          StringRef arg(*Arg);
+          if (arg.startswith("-ivfsoverlay")) {
+            // FIXME: This is a hack and calls for a better design.
+            //
+            // Filter out any -ivfsoverlay options that include an
+            // unextended-module-overlay.yaml overlay. By convention the Xcode
+            // buildsystem uses these while *building* mixed Objective-C and
+            // Swift frameworks; but they should never be used to *import* the
+            // module defined in the framework.
             auto Next = std::next(Arg);
             if (Next != E &&
                 StringRef(*Next).endswith("unextended-module-overlay.yaml")) {
               ++Arg;
               continue;
             }
+          } else if (arg.startswith("-fdebug-prefix-map=")) {
+            // We don't serialize the debug prefix map flags as these
+            // contain absoute paths that are not usable on different
+            // machines. These flags are not necessary to compile the
+            // clang modules again so are safe to remove.
+            continue;
           }
-          XCC.emit(ScratchRecord, *Arg);
+          XCC.emit(ScratchRecord, arg);
         }
       }
     }
@@ -1131,14 +1142,16 @@ void Serializer::writeInputBlock(const SerializationOptions &options) {
   input_block::ModuleInterfaceLayout ModuleInterface(Out);
 
   if (options.SerializeOptionsForDebugging) {
+    const auto &PathMapper = options.DebuggingOptionsPrefixMap;
     const SearchPathOptions &searchPathOpts = M->getASTContext().SearchPathOpts;
     // Put the framework search paths first so that they'll be preferred upon
     // deserialization.
     for (auto &framepath : searchPathOpts.FrameworkSearchPaths)
       SearchPath.emit(ScratchRecord, /*framework=*/true, framepath.IsSystem,
-                      framepath.Path);
+                      PathMapper.remapPath(framepath.Path));
     for (auto &path : searchPathOpts.ImportSearchPaths)
-      SearchPath.emit(ScratchRecord, /*framework=*/false, /*system=*/false, path);
+      SearchPath.emit(ScratchRecord, /*framework=*/false, /*system=*/false,
+                      PathMapper.remapPath(path));
   }
 
   // Note: We're not using StringMap here because we don't need to own the
@@ -1472,7 +1485,7 @@ void Serializer::writeASTBlockEntity(const SILLayout *layout) {
       typeRef |= 0x80000000U;
     data.push_back(typeRef);
   }
-  
+
   unsigned abbrCode
     = DeclTypeAbbrCodes[SILLayoutLayout::Code];
 
@@ -1707,7 +1720,7 @@ static bool shouldSerializeMember(Decl *D) {
 
   case DeclKind::OpaqueType:
     return true;
-      
+
   case DeclKind::EnumElement:
   case DeclKind::Protocol:
   case DeclKind::Constructor:
@@ -1805,14 +1818,14 @@ void Serializer::writeCrossReference(const DeclContext *DC, uint32_t pathLen) {
     if (auto opaque = dyn_cast<OpaqueTypeDecl>(generic)) {
       if (!opaque->hasName()) {
         abbrCode = DeclTypeAbbrCodes[XRefOpaqueReturnTypePathPieceLayout::Code];
-        
+
         XRefOpaqueReturnTypePathPieceLayout::emitRecord(Out, ScratchRecord,
                   abbrCode,
                   addDeclBaseNameRef(opaque->getOpaqueReturnTypeIdentifier()));
         break;
       }
     }
-      
+
     assert(generic->hasName());
 
     abbrCode = DeclTypeAbbrCodes[XRefTypePathPieceLayout::Code];
@@ -1853,7 +1866,7 @@ void Serializer::writeCrossReference(const DeclContext *DC, uint32_t pathLen) {
   case DeclContextKind::SubscriptDecl: {
     auto SD = cast<SubscriptDecl>(DC);
     writeCrossReference(DC->getParent(), pathLen + 1);
-    
+
     Type ty = SD->getInterfaceType()->getCanonicalType();
 
     abbrCode = DeclTypeAbbrCodes[XRefValuePathPieceLayout::Code];
@@ -1864,7 +1877,7 @@ void Serializer::writeCrossReference(const DeclContext *DC, uint32_t pathLen) {
                                          SD->isStatic());
     break;
   }
-      
+
   case DeclContextKind::AbstractFunctionDecl: {
     if (auto fn = dyn_cast<AccessorDecl>(DC)) {
       auto storage = fn->getStorage();
@@ -1976,7 +1989,7 @@ void Serializer::writeCrossReference(const Decl *D) {
                    addDeclBaseNameRef(opaque->getOpaqueReturnTypeIdentifier()));
     return;
   }
-  
+
   if (auto genericParam = dyn_cast<GenericTypeParamDecl>(D)) {
     assert(!D->getDeclContext()->isModuleScopeContext() &&
            "Cannot cross reference a generic type decl at module scope.");
@@ -4677,7 +4690,7 @@ class ClangToSwiftBasicWriter :
 
   Serializer &S;
   SmallVectorImpl<uint64_t> &Record;
-  using TypeWriter = 
+  using TypeWriter =
     clang::serialization::AbstractTypeWriter<ClangToSwiftBasicWriter>;
   TypeWriter Types;
 
@@ -5481,7 +5494,7 @@ void Serializer::writeAST(ModuleOrSourceFile DC) {
                                              /*isLocal=*/true);
       }
     }
-    
+
     for (auto OTD : opaqueReturnTypeDecls) {
       // FIXME: We should delay parsing function bodies so these type decls
       //        don't even get added to the file.