[Caching] Create clang importer from cc1 args directly

When caching build is enabled, teach dependency scanner to report
command-lines with `-direct-clang-cc1-module-build` so the later
compilation can instantiate clang importer with cc1 args directly. This
avoids running clang driver code, which might involve file system
lookups, which are the file deps that are not captured and might result
in different compilation mode.

rdar://119275464

Author: cachemeifyoucan

include/swift/ClangImporter/ClangImporter.h

@@ -514,6 +514,9 @@ class ClangImporter final : public ClangModuleLoader {
 
   std::string getClangModuleHash() const;
 
+  /// Get clang import creation cc1 args for swift explicit module build.
+  std::vector<std::string> getSwiftExplicitModuleDirectCC1Args() const;
+
   /// If we already imported a given decl successfully, return the corresponding
   /// Swift decl as an Optional<Decl *>, but if we previously tried and failed
   /// to import said decl then return nullptr.

lib/ClangImporter/ClangImporter.cpp

@@ -58,7 +58,9 @@
 #include "clang/Basic/Module.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/Version.h"
+#include "clang/CAS/CASOptions.h"
 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
+#include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Frontend/TextDiagnosticPrinter.h"
 #include "clang/Frontend/Utils.h"
@@ -73,8 +75,10 @@
 #include "clang/Sema/Sema.h"
 #include "clang/Serialization/ASTReader.h"
 #include "clang/Serialization/ASTWriter.h"
+#include "clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/CrashRecoveryContext.h"
 #include "llvm/Support/FileCollector.h"
@@ -1067,26 +1071,7 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
     ClangImporter *importer, ASTContext &ctx,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
     bool ignoreClangTarget) {
-  // If using direct cc1 module build, return extra args only.
-  if (ctx.ClangImporterOpts.DirectClangCC1ModuleBuild)
-    return ctx.ClangImporterOpts.ExtraArgs;
-
-  // Otherwise, create cc1 arguments from driver args.
-  auto driverArgs = getClangDriverArguments(ctx, ignoreClangTarget);
-
-  llvm::SmallVector<const char *> invocationArgs;
-  invocationArgs.reserve(driverArgs.size());
-  llvm::for_each(driverArgs, [&](const std::string &Arg) {
-    invocationArgs.push_back(Arg.c_str());
-  });
-
-  if (ctx.ClangImporterOpts.DumpClangDiagnostics) {
-    llvm::errs() << "clang importer driver args: '";
-    llvm::interleave(
-        invocationArgs, [](StringRef arg) { llvm::errs() << arg; },
-        [] { llvm::errs() << "' '"; });
-    llvm::errs() << "'\n";
-  }
+  std::unique_ptr<clang::CompilerInvocation> CI;
 
   // Set up a temporary diagnostic client to report errors from parsing the
   // command line, which may be important for Swift clients if, for example,
@@ -1098,24 +1083,60 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
   // clang::CompilerInstance is created.
   llvm::IntrusiveRefCntPtr<clang::DiagnosticOptions> tempDiagOpts{
       new clang::DiagnosticOptions};
-
   auto *tempDiagClient =
       new ClangDiagnosticConsumer(importer->Impl, *tempDiagOpts,
                                   ctx.ClangImporterOpts.DumpClangDiagnostics);
-
   auto clangDiags = clang::CompilerInstance::createDiagnostics(
       tempDiagOpts.get(), tempDiagClient,
       /*owned*/ true);
 
-  clang::CreateInvocationOptions CIOpts;
-  CIOpts.VFS = VFS;
-  CIOpts.Diags = clangDiags;
-  CIOpts.RecoverOnError = false;
-  CIOpts.ProbePrecompiled = true;
-  auto CI = clang::createInvocation(invocationArgs, std::move(CIOpts));
+  // If using direct cc1 module build, use extra args to setup ClangImporter.
+  if (ctx.ClangImporterOpts.DirectClangCC1ModuleBuild) {
+    llvm::SmallVector<const char *> clangArgs;
+    clangArgs.reserve(ctx.ClangImporterOpts.ExtraArgs.size());
+    llvm::for_each(
+        ctx.ClangImporterOpts.ExtraArgs,
+        [&](const std::string &Arg) { clangArgs.push_back(Arg.c_str()); });
+
+    // Try parse extra args, if failed, return nullopt.
+    CI = std::make_unique<clang::CompilerInvocation>();
+    if (!clang::CompilerInvocation::CreateFromArgs(*CI, clangArgs,
+                                                   *clangDiags))
+      return std::nullopt;
 
-  if (!CI)
-    return std::nullopt;
+    // Forwards some options from swift to clang even using direct mode. This is
+    // to reduce the number of argument passing on the command-line and swift
+    // compiler can be more efficient to compute swift cache key without having
+    // the knowledge about clang command-line options.
+    if (ctx.CASOpts.EnableCaching)
+      CI->getCASOpts() = ctx.CASOpts.CASOpts;
+  } else {
+    // Otherwise, create cc1 arguments from driver args.
+    auto driverArgs = getClangDriverArguments(ctx, ignoreClangTarget);
+
+    llvm::SmallVector<const char *> invocationArgs;
+    invocationArgs.reserve(driverArgs.size());
+    llvm::for_each(driverArgs, [&](const std::string &Arg) {
+      invocationArgs.push_back(Arg.c_str());
+    });
+
+    if (ctx.ClangImporterOpts.DumpClangDiagnostics) {
+      llvm::errs() << "clang importer driver args: '";
+      llvm::interleave(
+          invocationArgs, [](StringRef arg) { llvm::errs() << arg; },
+          [] { llvm::errs() << "' '"; });
+      llvm::errs() << "'\n";
+    }
+
+    clang::CreateInvocationOptions CIOpts;
+    CIOpts.VFS = VFS;
+    CIOpts.Diags = clangDiags;
+    CIOpts.RecoverOnError = false;
+    CIOpts.ProbePrecompiled = true;
+    CI = clang::createInvocation(invocationArgs, std::move(CIOpts));
+    if (!CI)
+      return std::nullopt;
+  }
 
   // FIXME: clang fails to generate a module if there is a `-fmodule-map-file`
   // argument pointing to a missing file.
@@ -3919,6 +3940,63 @@ std::string ClangImporter::getClangModuleHash() const {
   return Impl.Invocation->getModuleHash(Impl.Instance->getDiagnostics());
 }
 
+std::vector<std::string>
+ClangImporter::getSwiftExplicitModuleDirectCC1Args() const {
+  llvm::SmallVector<const char*> clangArgs;
+  clangArgs.reserve(Impl.ClangArgs.size());
+  llvm::for_each(Impl.ClangArgs, [&](const std::string &Arg) {
+    clangArgs.push_back(Arg.c_str());
+  });
+
+  clang::CompilerInvocation instance;
+  clang::DiagnosticsEngine clangDiags(new clang::DiagnosticIDs(),
+                                      new clang::DiagnosticOptions(),
+                                      new clang::IgnoringDiagConsumer());
+  bool success = clang::CompilerInvocation::CreateFromArgs(instance, clangArgs,
+                                                           clangDiags);
+  (void)success;
+  assert(success && "clang options from clangImporter failed to parse");
+
+  if (!Impl.SwiftContext.CASOpts.EnableCaching)
+    return instance.getCC1CommandLine();
+
+  // Clear some options that are not needed.
+  instance.clearImplicitModuleBuildOptions();
+
+  // CASOpts are forwarded from swift arguments.
+  instance.getCASOpts() = clang::CASOptions();
+
+  // HeaderSearchOptions.
+  // Clang search options are only used by scanner and clang importer from main
+  // module should not using search paths to find modules.
+  auto &HSOpts = instance.getHeaderSearchOpts();
+  HSOpts.VFSOverlayFiles.clear();
+  HSOpts.UserEntries.clear();
+  HSOpts.SystemHeaderPrefixes.clear();
+
+  // FrontendOptions.
+  auto &FEOpts = instance.getFrontendOpts();
+  FEOpts.IncludeTimestamps = false;
+  FEOpts.ModuleMapFiles.clear();
+
+  // PreprocessorOptions.
+  // Cannot clear macros as the main module clang importer doesn't have clang
+  // include tree created and it has to be created from command-line. However,
+  // include files are no collected into CASFS so they will not be found so
+  // clear them to avoid problem.
+  auto &PPOpts = instance.getPreprocessorOpts();
+  PPOpts.MacroIncludes.clear();
+  PPOpts.Includes.clear();
+
+  if (Impl.SwiftContext.ClangImporterOpts.UseClangIncludeTree) {
+    // FileSystemOptions.
+    auto &FSOpts = instance.getFileSystemOpts();
+    FSOpts.WorkingDir.clear();
+  }
+
+  return instance.getCC1CommandLine();
+}
+
 std::optional<Decl *>
 ClangImporter::importDeclCached(const clang::NamedDecl *ClangDecl) {
   return Impl.importDeclCached(ClangDecl, Impl.CurrentVersion);

lib/DependencyScan/ModuleDependencyScanner.cpp

@@ -346,6 +346,7 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(
         {"-Xcc", "-target", "-Xcc", ScanASTContext.LangOpts.Target.str()});
 
   std::string rootID;
+  std::vector<std::string> buildArgs;
   if (tracker) {
     tracker->startTracking();
     for (auto fileUnit : mainModule->getFiles()) {
@@ -371,10 +372,22 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(
       return std::make_error_code(std::errc::io_error);
     }
     rootID = root->getID().toString();
+
+    buildArgs.push_back("-direct-clang-cc1-module-build");
+    for (auto &arg : clangImporter->getSwiftExplicitModuleDirectCC1Args()) {
+      buildArgs.push_back("-Xcc");
+      buildArgs.push_back(arg);
+    }
   }
 
-  auto mainDependencies =
-      ModuleDependencyInfo::forSwiftSourceModule(rootID, {}, {}, ExtraPCMArgs);
+  llvm::SmallVector<StringRef> buildCommands;
+  buildCommands.reserve(buildArgs.size());
+  llvm::for_each(buildArgs, [&](const std::string &arg) {
+    buildCommands.emplace_back(arg);
+  });
+
+  auto mainDependencies = ModuleDependencyInfo::forSwiftSourceModule(
+      rootID, buildCommands, {}, ExtraPCMArgs);
 
   llvm::StringSet<> alreadyAddedModules;
   // Compute Implicit dependencies of the main module

lib/DependencyScan/ScanDependencies.cpp

@@ -290,19 +290,11 @@ static llvm::Error resolveExplicitModuleInputs(
         }
       }
       if (!clangDepDetails->moduleCacheKey.empty()) {
-        auto appendXclang = [&]() {
-          if (!resolvingDepInfo.isClangModule()) {
-            // clang module build using cc1 arg so this is not needed.
-            commandLine.push_back("-Xcc");
-            commandLine.push_back("-Xclang");
-          }
-          commandLine.push_back("-Xcc");
-        };
-        appendXclang();
+        commandLine.push_back("-Xcc");
         commandLine.push_back("-fmodule-file-cache-key");
-        appendXclang();
+        commandLine.push_back("-Xcc");
         commandLine.push_back(clangDepDetails->mappedPCMPath);
-        appendXclang();
+        commandLine.push_back("-Xcc");
         commandLine.push_back(clangDepDetails->moduleCacheKey);
       }
 

lib/Frontend/CompilerInvocation.cpp

@@ -1659,43 +1659,6 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts, ArgList &Args,
     Opts.ExtraArgs.push_back(A->getValue());
   }
 
-  for (const Arg *A : Args.filtered(OPT_file_prefix_map,
-                                    OPT_debug_prefix_map)) {
-    std::string Val(A->getValue());
-    // Forward -debug-prefix-map arguments from Swift to Clang as
-    // -fdebug-prefix-map= and -file-prefix-map as -ffile-prefix-map=.
-    //
-    // This is required to ensure DIFiles created there, like
-    /// "<swift-imported-modules>", as well as index data, have their paths
-    // remapped properly.
-    //
-    // (Note, however, that Clang's usage of std::map means that the remapping
-    // may not be applied in the same order, which can matter if one mapping is
-    // a prefix of another.)
-    if (A->getOption().matches(OPT_file_prefix_map))
-      Opts.ExtraArgs.push_back("-ffile-prefix-map=" + Val);
-    else
-      Opts.ExtraArgs.push_back("-fdebug-prefix-map=" + Val);
-  }
-
-  if (auto *A = Args.getLastArg(OPT_file_compilation_dir)) {
-    // Forward the -file-compilation-dir flag to correctly set the
-    // debug compilation directory.
-    std::string Val(A->getValue());
-    Opts.ExtraArgs.push_back("-ffile-compilation-dir=" + Val);
-  }
-
-  if (CASOpts.CASFSRootIDs.empty() &&
-      CASOpts.ClangIncludeTrees.empty()) {
-    if (!workingDirectory.empty()) {
-      // Provide a working directory to Clang as well if there are any -Xcc
-      // options, in case some of them are search-related. But do it at the
-      // beginning, so that an explicit -Xcc -working-directory will win.
-      Opts.ExtraArgs.insert(Opts.ExtraArgs.begin(),
-                            {"-working-directory", workingDirectory.str()});
-    }
-  }
-
   Opts.DumpClangDiagnostics |= Args.hasArg(OPT_dump_clang_diagnostics);
 
   // When the repl is invoked directly (ie. `lldb --repl="..."`) the action
@@ -1755,6 +1718,45 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts, ArgList &Args,
     Opts.HasClangIncludeTreeRoot |= Args.hasArg(OPT_clang_include_tree_root);
   }
 
+  // If in direct clang cc1 module build mode, return early.
+  if (Opts.DirectClangCC1ModuleBuild)
+    return false;
+
+  // Only amend the following path option when not in direct cc1 mode.
+  for (const Arg *A : Args.filtered(OPT_file_prefix_map,
+                                    OPT_debug_prefix_map)) {
+    std::string Val(A->getValue());
+    // Forward -debug-prefix-map arguments from Swift to Clang as
+    // -fdebug-prefix-map= and -file-prefix-map as -ffile-prefix-map=.
+    //
+    // This is required to ensure DIFiles created there, like
+    /// "<swift-imported-modules>", as well as index data, have their paths
+    // remapped properly.
+    //
+    // (Note, however, that Clang's usage of std::map means that the remapping
+    // may not be applied in the same order, which can matter if one mapping is
+    // a prefix of another.)
+    if (A->getOption().matches(OPT_file_prefix_map))
+      Opts.ExtraArgs.push_back("-ffile-prefix-map=" + Val);
+    else
+      Opts.ExtraArgs.push_back("-fdebug-prefix-map=" + Val);
+  }
+
+  if (auto *A = Args.getLastArg(OPT_file_compilation_dir)) {
+    // Forward the -file-compilation-dir flag to correctly set the
+    // debug compilation directory.
+    std::string Val(A->getValue());
+    Opts.ExtraArgs.push_back("-ffile-compilation-dir=" + Val);
+  }
+
+  if (!workingDirectory.empty()) {
+    // Provide a working directory to Clang as well if there are any -Xcc
+    // options, in case some of them are search-related. But do it at the
+    // beginning, so that an explicit -Xcc -working-directory will win.
+    Opts.ExtraArgs.insert(Opts.ExtraArgs.begin(),
+                          {"-working-directory", workingDirectory.str()});
+  }
+
   return false;
 }