[Dependency Scanning] Bridge Clang dependency scanner results on-demand

Instead of always bridging all of the discovered modules of all of the queries, only do so for modules which are not already cached

Author: artemcm

include/swift/AST/ModuleDependencies.h

@@ -23,9 +23,11 @@
 #include "swift/Basic/Assertions.h"
 #include "swift/Basic/CXXStdlibKind.h"
 #include "swift/Basic/LLVM.h"
+#include "swift/ClangImporter/ClangImporter.h"
 #include "swift/Serialization/Validation.h"
 #include "clang/CAS/CASOptions.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
@@ -174,6 +176,18 @@ void registerCxxInteropLibraries(
     const llvm::Triple &Target, StringRef mainModuleName, bool hasStaticCxx,
     bool hasStaticCxxStdlib, CXXStdlibKind cxxStdlibKind,
     std::function<void(const LinkLibrary &)> RegistrationCallback);
+
+using RemapPathCallback = llvm::function_ref<std::string(StringRef)>;
+using LookupModuleOutputCallback =
+    llvm::function_ref<std::string(const clang::tooling::dependencies::ModuleDeps &,
+                                   clang::tooling::dependencies::ModuleOutputKind)>;
+
+ModuleDependencyInfo
+bridgeClangModuleDependency(
+    const ASTContext &ctx,
+    const clang::tooling::dependencies::ModuleDeps &clangDependency,
+    LookupModuleOutputCallback LookupModuleOutput,
+    RemapPathCallback remapPath = nullptr);
 } // namespace dependencies
 
 struct ScannerImportStatementInfo {
@@ -1178,8 +1192,11 @@ class ModuleDependenciesCache {
                         ModuleDependencyInfo dependencies);
 
   /// Record dependencies for the given collection of Clang modules.
-  void recordClangDependencies(ModuleDependencyVector moduleDependencies,
-                               DiagnosticEngine &diags);
+  void recordClangDependencies(
+      const clang::tooling::dependencies::ModuleDepsGraph &dependencies,
+      const ASTContext &ctx,
+      dependencies::LookupModuleOutputCallback LookupModuleOutput,
+      dependencies::RemapPathCallback remapPath = nullptr);
 
   /// Update stored dependencies for the given module.
   void updateDependency(ModuleDependencyID moduleID,

include/swift/ClangImporter/ClangImporter.h

@@ -492,19 +492,6 @@ class ClangImporter final : public ClangModuleLoader {
 
   void verifyAllModules() override;
 
-  using RemapPathCallback = llvm::function_ref<std::string(StringRef)>;
-  using LookupModuleOutputCallback =
-      llvm::function_ref<std::string(const clang::tooling::dependencies::ModuleDeps &,
-                                     clang::tooling::dependencies::ModuleOutputKind)>;
-
-  static llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
-  bridgeClangModuleDependencies(
-      const ASTContext &ctx,
-      clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
-      clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
-      LookupModuleOutputCallback LookupModuleOutput,
-      RemapPathCallback remapPath = nullptr);
-
   static void getBridgingHeaderOptions(
       const ASTContext &ctx,
       const clang::tooling::dependencies::TranslationUnitDeps &deps,

include/swift/DependencyScan/ModuleDependencyScanner.h

@@ -40,7 +40,8 @@ class ModuleDependencyScanningWorker {
 
 private:
   /// Retrieve the module dependencies for the Clang module with the given name.
-  ClangModuleScannerQueryResult scanFilesystemForClangModuleDependency(
+  std::optional<clang::tooling::dependencies::TranslationUnitDeps>
+  scanFilesystemForClangModuleDependency(
       Identifier moduleName,
       const llvm::DenseSet<clang::tooling::dependencies::ModuleID>
           &alreadySeenModules);

include/swift/Serialization/ScanningLoaders.h

@@ -40,18 +40,6 @@ struct SwiftModuleScannerQueryResult {
   std::vector<IncompatibleCandidate> incompatibleCandidates;
 };
 
-/// Result of looking up a Clang module on the current filesystem
-/// search paths.
-struct ClangModuleScannerQueryResult {
-  ClangModuleScannerQueryResult(const ModuleDependencyVector &dependencyModuleGraph,
-                                const std::vector<std::string> &visibleModuleIdentifiers)
-      : foundDependencyModuleGraph(dependencyModuleGraph),
-        visibleModuleIdentifiers(visibleModuleIdentifiers) {}
-
-  ModuleDependencyVector foundDependencyModuleGraph;
-  std::vector<std::string> visibleModuleIdentifiers;
-};
-
 /// A module "loader" that looks for .swiftinterface and .swiftmodule files
 /// for the purpose of determining dependencies, but does not attempt to
 /// load the module files.

lib/AST/ModuleDependencies.cpp

@@ -23,6 +23,7 @@
 #include "swift/AST/SourceFile.h"
 #include "swift/Frontend/Frontend.h"
 #include "swift/Strings.h"
+#include "clang/Lex/HeaderSearchOptions.h"
 #include "llvm/Config/config.h"
 #include "llvm/Support/Path.h"
 using namespace swift;
@@ -598,6 +599,120 @@ void swift::dependencies::registerCxxInteropLibraries(
   }
 }
 
+ModuleDependencyInfo swift::dependencies::bridgeClangModuleDependency(
+    const ASTContext &ctx,
+    const clang::tooling::dependencies::ModuleDeps &clangModuleDep,
+    LookupModuleOutputCallback lookupModuleOutput, RemapPathCallback callback) {
+  auto remapPath = [&](StringRef path) {
+    if (callback)
+      return callback(path);
+    return path.str();
+  };
+
+  // File dependencies for this module.
+  std::vector<std::string> fileDeps;
+  clangModuleDep.forEachFileDep(
+      [&fileDeps](StringRef fileDep) { fileDeps.emplace_back(fileDep); });
+
+  std::vector<std::string> swiftArgs;
+  auto addClangArg = [&](Twine arg) {
+    swiftArgs.push_back("-Xcc");
+    swiftArgs.push_back(arg.str());
+  };
+
+  // We are using Swift frontend mode.
+  swiftArgs.push_back("-frontend");
+
+  // Swift frontend action: -emit-pcm
+  swiftArgs.push_back("-emit-pcm");
+  swiftArgs.push_back("-module-name");
+  swiftArgs.push_back(clangModuleDep.ID.ModuleName);
+
+  auto pcmPath = lookupModuleOutput(
+      clangModuleDep,
+      clang::tooling::dependencies::ModuleOutputKind::ModuleFile);
+  swiftArgs.push_back("-o");
+  swiftArgs.push_back(pcmPath);
+
+  // Ensure that the resulting PCM build invocation uses Clang frontend
+  // directly
+  swiftArgs.push_back("-direct-clang-cc1-module-build");
+
+  // Swift frontend option for input file path (Foo.modulemap).
+  swiftArgs.push_back(remapPath(clangModuleDep.ClangModuleMapFile));
+
+  auto invocation = clangModuleDep.getUnderlyingCompilerInvocation();
+  // Clear some options from clang scanner.
+  invocation.getMutFrontendOpts().ModuleCacheKeys.clear();
+  invocation.getMutFrontendOpts().PathPrefixMappings.clear();
+  invocation.getMutFrontendOpts().OutputFile.clear();
+
+  // Reset CASOptions since that should be coming from swift.
+  invocation.getMutCASOpts() = clang::CASOptions();
+  invocation.getMutFrontendOpts().CASIncludeTreeID.clear();
+
+  // FIXME: workaround for rdar://105684525: find the -ivfsoverlay option
+  // from clang scanner and pass to swift.
+  if (!ctx.CASOpts.EnableCaching) {
+    auto &overlayFiles = invocation.getMutHeaderSearchOpts().VFSOverlayFiles;
+    for (auto overlay : overlayFiles) {
+      swiftArgs.push_back("-vfsoverlay");
+      swiftArgs.push_back(overlay);
+    }
+  }
+
+  // Add args reported by the scanner.
+  auto clangArgs = invocation.getCC1CommandLine();
+  llvm::for_each(clangArgs, addClangArg);
+
+  // CASFileSystemRootID.
+  std::string RootID = clangModuleDep.CASFileSystemRootID
+                           ? clangModuleDep.CASFileSystemRootID->toString()
+                           : "";
+
+  std::string IncludeTree =
+      clangModuleDep.IncludeTreeID ? *clangModuleDep.IncludeTreeID : "";
+
+  ctx.CASOpts.enumerateCASConfigurationFlags(
+      [&](StringRef Arg) { swiftArgs.push_back(Arg.str()); });
+
+  if (!IncludeTree.empty()) {
+    swiftArgs.push_back("-clang-include-tree-root");
+    swiftArgs.push_back(IncludeTree);
+  }
+  std::string mappedPCMPath = remapPath(pcmPath);
+
+  std::vector<LinkLibrary> LinkLibraries;
+  for (const auto &ll : clangModuleDep.LinkLibraries)
+    LinkLibraries.emplace_back(ll.Library,
+                               ll.IsFramework ? LibraryKind::Framework
+                                              : LibraryKind::Library,
+                               /*static=*/false);
+
+  // Module-level dependencies.
+  llvm::StringSet<> alreadyAddedModules;
+  auto bridgedDependencyInfo = ModuleDependencyInfo::forClangModule(
+      pcmPath, mappedPCMPath, clangModuleDep.ClangModuleMapFile,
+      clangModuleDep.ID.ContextHash, swiftArgs, fileDeps, LinkLibraries, RootID,
+      IncludeTree, /*module-cache-key*/ "", clangModuleDep.IsSystem);
+
+  std::vector<ModuleDependencyID> directDependencyIDs;
+  for (const auto &moduleName : clangModuleDep.ClangModuleDeps) {
+    // FIXME: This assumes, conservatively, that all Clang module imports
+    // are exported. We need to fix this once the clang scanner gains the
+    // appropriate API to query this.
+    bridgedDependencyInfo.addModuleImport(
+        moduleName.ModuleName, /* isExported */ true, AccessLevel::Public,
+        &alreadyAddedModules);
+    // It is safe to assume that all dependencies of a Clang module are Clang
+    // modules.
+    directDependencyIDs.push_back(
+        {moduleName.ModuleName, ModuleDependencyKind::Clang});
+  }
+  bridgedDependencyInfo.setImportedClangDependencies(directDependencyIDs);
+  return bridgedDependencyInfo;
+}
+
 void
 swift::dependencies::registerBackDeployLibraries(
     const IRGenOptions &IRGenOpts,
@@ -779,16 +894,19 @@ void ModuleDependenciesCache::recordDependency(
 }
 
 void ModuleDependenciesCache::recordClangDependencies(
-    ModuleDependencyVector dependencies, DiagnosticEngine &diags) {
+    const clang::tooling::dependencies::ModuleDepsGraph &dependencies,
+    const ASTContext &ctx,
+    dependencies::LookupModuleOutputCallback lookupModuleOutput,
+    dependencies::RemapPathCallback remapPath) {
   for (const auto &dep : dependencies) {
-    ASSERT(dep.first.Kind == ModuleDependencyKind::Clang);
-    auto newClangModuleDetails = dep.second.getAsClangModule();
-    if (hasDependency(dep.first)) {
+    auto depID =
+        ModuleDependencyID{dep.ID.ModuleName, ModuleDependencyKind::Clang};
+    if (hasDependency(depID)) {
       auto priorClangModuleDetails =
-          findKnownDependency(dep.first).getAsClangModule();
-      DEBUG_ASSERT(priorClangModuleDetails && newClangModuleDetails);
+          findKnownDependency(depID).getAsClangModule();
+      DEBUG_ASSERT(priorClangModuleDetails);
       auto priorContextHash = priorClangModuleDetails->contextHash;
-      auto newContextHash = newClangModuleDetails->contextHash;
+      auto newContextHash = dep.ID.ContextHash;
       if (priorContextHash != newContextHash) {
         // This situation means that within the same scanning action, Clang
         // Dependency Scanner has produced two different variants of the same
@@ -799,28 +917,33 @@ void ModuleDependenciesCache::recordClangDependencies(
         //
         // Emit a failure diagnostic here that is hopefully more actionable
         // for the time being.
-        diags.diagnose(SourceLoc(), diag::dependency_scan_unexpected_variant,
-                       dep.first.ModuleName);
-        diags.diagnose(
+        ctx.Diags.diagnose(SourceLoc(),
+                           diag::dependency_scan_unexpected_variant,
+                           dep.ID.ModuleName);
+        ctx.Diags.diagnose(
             SourceLoc(),
             diag::dependency_scan_unexpected_variant_context_hash_note,
             priorContextHash, newContextHash);
-        diags.diagnose(
+        ctx.Diags.diagnose(
             SourceLoc(),
             diag::dependency_scan_unexpected_variant_module_map_note,
-            priorClangModuleDetails->moduleMapFile,
-            newClangModuleDetails->moduleMapFile);
+            priorClangModuleDetails->moduleMapFile, dep.ClangModuleMapFile);
+
+        auto newClangModuleDetails =
+            dependencies::bridgeClangModuleDependency(
+                ctx, dep, lookupModuleOutput, remapPath)
+                .getAsClangModule();
 
         auto diagnoseExtraCommandLineFlags =
-            [&diags](const ClangModuleDependencyStorage *checkModuleDetails,
-                     const ClangModuleDependencyStorage *baseModuleDetails,
-                     bool isNewlyDiscovered) -> void {
+            [&ctx](const ClangModuleDependencyStorage *checkModuleDetails,
+                   const ClangModuleDependencyStorage *baseModuleDetails,
+                   bool isNewlyDiscovered) -> void {
           std::unordered_set<std::string> baseCommandLineSet(
               baseModuleDetails->buildCommandLine.begin(),
               baseModuleDetails->buildCommandLine.end());
           for (const auto &checkArg : checkModuleDetails->buildCommandLine)
             if (baseCommandLineSet.find(checkArg) == baseCommandLineSet.end())
-              diags.diagnose(
+              ctx.Diags.diagnose(
                   SourceLoc(),
                   diag::dependency_scan_unexpected_variant_extra_arg_note,
                   isNewlyDiscovered, checkArg);
@@ -831,9 +954,10 @@ void ModuleDependenciesCache::recordClangDependencies(
                                       priorClangModuleDetails, false);
       }
     } else {
-      recordDependency(dep.first.ModuleName, dep.second);
-      addSeenClangModule(clang::tooling::dependencies::ModuleID{
-        dep.first.ModuleName, newClangModuleDetails->contextHash});
+      recordDependency(dep.ID.ModuleName,
+                       dependencies::bridgeClangModuleDependency(
+                           ctx, dep, lookupModuleOutput, remapPath));
+      addSeenClangModule(dep.ID);
     }
   }
 }