Merge pull request #62439 from artemcm/Part2-BetterDependencyCaching-FileSystemCache

[Dependency Scanning] Use persistent dependency scanning filesystem in the Swift dependency scanner

Author: artemcm

include/swift/AST/ModuleDependencies.h

@@ -35,6 +35,7 @@ class ClangModuleDependenciesCacheImpl;
 class SourceFile;
 class ASTContext;
 class Identifier;
+class CompilerInstance;
 
 /// Which kind of module dependencies we are looking for.
 enum class ModuleDependenciesKind : int8_t {
@@ -501,34 +502,40 @@ class GlobalModuleDependenciesCache {
     std::vector<ModuleDependencyID> AllModules;
 
     /// Dependencies for modules that have already been computed.
-    /// This maps a dependency kind to a map of a module's name to a Dependency object
+    /// This maps a dependency kind to a map of a module's name to a Dependency
+    /// object
     ModuleDependenciesKindMap ModuleDependenciesMap;
   };
 
-  /// The 'persistent' Clang dependency scanner service
-  clang::tooling::dependencies::DependencyScanningService clangScanningService;
+  /// The persistent Clang dependency scanner service
+  clang::tooling::dependencies::DependencyScanningService ClangScanningService;
+  /// The global file system cache.
+  Optional<
+      clang::tooling::dependencies::DependencyScanningFilesystemSharedCache>
+      SharedFilesystemCache;
 
-  /// All cached Swift source module dependencies, in the order in which they were encountered
+  /// All cached Swift source module dependencies, in the order in which they
+  /// were encountered
   std::vector<ModuleDependencyID> AllSourceModules;
-
-  /// Dependencies for all Swift source-based modules discovered. Each one is the main
-  /// module of a prior invocation of the scanner.
+  /// Dependencies for all Swift source-based modules discovered. Each one is
+  /// the main module of a prior invocation of the scanner.
   ModuleNameToDependencyMap SwiftSourceModuleDependenciesMap;
 
-  /// A map from a String representing the target triple of a scanner invocation to the corresponding
-  /// cached dependencies discovered so far when using this triple.
-  llvm::StringMap<std::unique_ptr<ContextSpecificGlobalCacheState>> ContextSpecificCacheMap;
+  /// A map from a String representing the target triple of a scanner invocation
+  /// to the corresponding cached dependencies discovered so far when using this
+  /// triple.
+  llvm::StringMap<std::unique_ptr<ContextSpecificGlobalCacheState>>
+      ContextSpecificCacheMap;
 
   /// The current context hash configuration
   Optional<std::string> CurrentContextHash;
-
-  /// The context hashes used by scanners using this cache, in the order in which they were used
+  /// The context hashes used by scanners using this cache, in the order in
+  /// which they were used
   std::vector<std::string> AllContextHashes;
 
   /// Retrieve the dependencies map that corresponds to the given dependency
   /// kind.
-  ModuleNameToDependencyMap &
-  getDependenciesMap(ModuleDependenciesKind kind);
+  ModuleNameToDependencyMap &getDependenciesMap(ModuleDependenciesKind kind);
   const ModuleNameToDependencyMap &
   getDependenciesMap(ModuleDependenciesKind kind) const;
 
@@ -539,6 +546,16 @@ class GlobalModuleDependenciesCache {
   operator=(const GlobalModuleDependenciesCache &) = delete;
   virtual ~GlobalModuleDependenciesCache() {}
 
+  /// Query the service's filesystem cache
+  clang::tooling::dependencies::DependencyScanningFilesystemSharedCache &
+  getSharedFilesystemCache() {
+    assert(SharedFilesystemCache && "Expected a shared cache");
+    return *SharedFilesystemCache;
+  }
+
+  /// Wrap the filesystem on the specified `CompilerInstance` with a
+  /// caching `DependencyScanningWorkerFilesystem`
+  void overlaySharedFilesystemCacheForCompilation(CompilerInstance &Instance);
 private:
   /// Enforce clients not being allowed to query this cache directly, it must be
   /// wrapped in an instance of `ModuleDependenciesCache`.
@@ -552,19 +569,21 @@ class GlobalModuleDependenciesCache {
 
   /// Return context hashes of all scanner invocations that have used
   /// this cache instance.
-  const std::vector<std::string>& getAllContextHashes() const {
+  const std::vector<std::string> &getAllContextHashes() const {
     return AllContextHashes;
   }
 
   /// Whether we have cached dependency information for the given module.
   bool hasDependencies(StringRef moduleName,
                        Optional<ModuleDependenciesKind> kind) const;
 
-  /// Return a pointer to the context-specific cache state of the current scanning action.
-  ContextSpecificGlobalCacheState* getCurrentCache() const;
+  /// Return a pointer to the context-specific cache state of the current
+  /// scanning action.
+  ContextSpecificGlobalCacheState *getCurrentCache() const;
 
   /// Return a pointer to the cache state of the specified context hash.
-  ContextSpecificGlobalCacheState* getCacheForScanningContextHash(StringRef scanningContextHash) const;
+  ContextSpecificGlobalCacheState *
+  getCacheForScanningContextHash(StringRef scanningContextHash) const;
 
   /// Look for source-based module dependency details
   Optional<ModuleDependencies>
@@ -585,10 +604,13 @@ class GlobalModuleDependenciesCache {
   const ModuleDependencies *updateDependencies(ModuleDependencyID moduleID,
                                                ModuleDependencies dependencies);
 
-  /// Reference the list of all module dependencies that are not source-based modules
-  /// (i.e. interface dependencies, binary dependencies, clang dependencies).
-  const std::vector<ModuleDependencyID> &getAllNonSourceModules(StringRef scanningContextHash) const {
-    auto contextSpecificCache = getCacheForScanningContextHash(scanningContextHash);
+  /// Reference the list of all module dependencies that are not source-based
+  /// modules (i.e. interface dependencies, binary dependencies, clang
+  /// dependencies).
+  const std::vector<ModuleDependencyID> &
+  getAllNonSourceModules(StringRef scanningContextHash) const {
+    auto contextSpecificCache =
+        getCacheForScanningContextHash(scanningContextHash);
     return contextSpecificCache->AllModules;
   }
 

include/swift/DependencyScan/ScanDependencies.h

@@ -31,10 +31,6 @@ class GlobalModuleDependenciesCache;
 
 namespace dependencies {
 
-//using CompilerArgInstanceCacheMap =
-//    llvm::StringMap<std::pair<std::unique_ptr<CompilerInstance>,
-//                              std::unique_ptr<ModuleDependenciesCache>>>;
-
 using CompilerArgInstanceCacheMap =
     llvm::StringMap<std::tuple<std::unique_ptr<CompilerInstance>,
                                std::unique_ptr<GlobalModuleDependenciesCache>,

lib/AST/ModuleDependencies.cpp

@@ -16,6 +16,7 @@
 #include "swift/AST/ModuleDependencies.h"
 #include "swift/AST/Decl.h"
 #include "swift/AST/SourceFile.h"
+#include "swift/Frontend/Frontend.h"
 using namespace swift;
 
 ModuleDependenciesStorageBase::~ModuleDependenciesStorageBase() {}
@@ -226,14 +227,26 @@ void ModuleDependencies::addBridgingModuleDependency(
 }
 
 GlobalModuleDependenciesCache::GlobalModuleDependenciesCache()
-  : clangScanningService(
-                         clang::tooling::dependencies::ScanningMode::DependencyDirectivesScan,
+  : ClangScanningService(clang::tooling::dependencies::ScanningMode::DependencyDirectivesScan,
                          clang::tooling::dependencies::ScanningOutputFormat::Full,
                          clang::CASOptions(),
                          /* Cache (llvm::cas::ActionCache) */ nullptr,
                          /* SharedFS */ nullptr,
                          /* ReuseFileManager */ false,
-                         /* OptimizeArgs */ false) {}
+                         /* OptimizeArgs */ false) {
+    SharedFilesystemCache.emplace();
+}
+
+void GlobalModuleDependenciesCache::overlaySharedFilesystemCacheForCompilation(CompilerInstance &Instance) {
+ auto existingFS = Instance.getSourceMgr().getFileSystem();
+ llvm::IntrusiveRefCntPtr<
+     clang::tooling::dependencies::DependencyScanningWorkerFilesystem>
+     depFS =
+         new clang::tooling::dependencies::DependencyScanningWorkerFilesystem(
+             getSharedFilesystemCache(), existingFS);
+ Instance.getSourceMgr().setFileSystem(depFS);
+}
+
 GlobalModuleDependenciesCache::ContextSpecificGlobalCacheState *
 GlobalModuleDependenciesCache::getCurrentCache() const {
   assert(CurrentContextHash.has_value() &&
@@ -429,7 +442,7 @@ ModuleDependenciesCache::ModuleDependenciesCache(
     std::string scanningContextHash)
     : globalCache(globalCache),
       mainScanModuleName(mainScanModuleName),
-      clangScanningTool(globalCache.clangScanningService) {
+      clangScanningTool(globalCache.ClangScanningService) {
   globalCache.configureForContextHash(scannerContextHash);
   for (auto kind = ModuleDependenciesKind::FirstKind;
        kind != ModuleDependenciesKind::LastKind; ++kind) {

lib/DependencyScan/DependencyScanningTool.cpp

@@ -217,6 +217,9 @@ DependencyScanningTool::initCompilerInstanceForScan(
   auto Instance = std::make_unique<CompilerInstance>();
   Instance->addDiagnosticConsumer(&CDC);
 
+  // Wrap the filesystem with a caching `DependencyScanningWorkerFilesystem`
+  SharedCache->overlaySharedFilesystemCacheForCompilation(*Instance);
+
   // Basic error checking on the arguments
   if (CommandArgs.empty()) {
     Instance->getDiags().diagnose(SourceLoc(), diag::error_no_frontend_args);

lib/DependencyScan/ScanDependencies.cpp

@@ -1242,13 +1242,13 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) {
   GlobalModuleDependenciesCache globalCache;
   if (opts.ReuseDependencyScannerCache)
     deserializeDependencyCache(instance, globalCache);
-
-  auto ModuleCachePath = getModuleCachePathFromClang(
-               Context.getClangModuleLoader()->getClangInstance());
-
+  // Wrap the filesystem with a caching `DependencyScanningWorkerFilesystem`
+  globalCache.overlaySharedFilesystemCacheForCompilation(instance);
   ModuleDependenciesCache cache(globalCache,
                                 instance.getMainModule()->getNameStr().str(),
                                 instance.getInvocation().getModuleScanningHash());
+  auto ModuleCachePath = getModuleCachePathFromClang(
+               Context.getClangModuleLoader()->getClangInstance());
 
   // Execute scan
   auto dependenciesOrErr = performModuleScan(instance, cache);
@@ -1312,6 +1312,9 @@ bool swift::dependencies::batchScanDependencies(
   // The primary cache used for scans carried out with the compiler instance
   // we have created
   GlobalModuleDependenciesCache singleUseGlobalCache;
+  // Wrap the filesystem with a caching `DependencyScanningWorkerFilesystem`
+  // Wrap the filesystem with a caching `DependencyScanningWorkerFilesystem`
+  singleUseGlobalCache.overlaySharedFilesystemCacheForCompilation(instance);
   ModuleDependenciesCache cache(singleUseGlobalCache,
                                 instance.getMainModule()->getNameStr().str(),
                                 instance.getInvocation().getModuleScanningHash());