Merge pull request #62411 from artemcm/BetterDependencyCaching-ContextHash

[Dependency Scanning] Simplify the persistent dependency scanning cache

Author: artemcm

include/swift/AST/IRGenOptions.h

@@ -554,6 +554,12 @@ class IRGenOptions {
     return llvm::hash_value(0);
   }
 
+  /// Return a hash code of any components from these options that should
+  /// contribute to a Swift Dependency Scanning hash.
+  llvm::hash_code getModuleScanningHashComponents() const {
+    return llvm::hash_value(0);
+  }
+
   bool hasMultipleIRGenThreads() const { return !UseSingleModuleLLVMEmission && NumThreads > 1; }
   bool shouldPerformIRGenerationInParallel() const { return !UseSingleModuleLLVMEmission && NumThreads != 0; }
   bool hasMultipleIGMs() const { return hasMultipleIRGenThreads(); }

include/swift/AST/ModuleDependencies.h

@@ -76,12 +76,6 @@ struct ModuleDependenciesKindHash {
   }
 };
 
-/// Details of a given module used for dependency scanner cache queries.
-struct ModuleLookupSpecifics {
-  Optional<ModuleDependenciesKind> kind;
-  llvm::StringSet<> currentSearchPaths;
-};
-
 /// Base class for the variant storage of ModuleDependencies.
 ///
 /// This class is mostly an implementation detail for \c ModuleDependencies.
@@ -331,6 +325,7 @@ class ModuleDependencies {
     : storage(std::move(storage)) { }
 
 public:
+  ModuleDependencies() = default;
   ModuleDependencies(const ModuleDependencies &other)
     : storage(other.storage->clone()) { }
   ModuleDependencies(ModuleDependencies &&other) = default;
@@ -481,9 +476,10 @@ class ModuleDependencies {
 
 using ModuleDependencyID = std::pair<std::string, ModuleDependenciesKind>;
 using ModuleDependenciesVector = llvm::SmallVector<ModuleDependencies, 1>;
+using ModuleNameToDependencyMap = llvm::StringMap<ModuleDependencies>;
 using ModuleDependenciesKindMap =
     std::unordered_map<ModuleDependenciesKind,
-                       llvm::StringMap<ModuleDependenciesVector>,
+                       ModuleNameToDependencyMap,
                        ModuleDependenciesKindHash>;
 using ModuleDependenciesKindRefMap =
     std::unordered_map<ModuleDependenciesKind,
@@ -497,20 +493,15 @@ using ModuleDependenciesKindRefMap =
 /// `DependencyScanningTool`). It is not to be queried directly, but is rather
 /// meant to be wrapped in an instance of `ModuleDependenciesCache`, responsible
 /// for recording new dependencies and answering cache queries in a given scan.
-/// Queries to this cache must be disambiguated with a set of search paths to
-/// ensure that the returned cached dependency was one that can be found in the
-/// current scanning action's filesystem view.
 class GlobalModuleDependenciesCache {
-  /// Global cache contents specific to a target-triple specified on a scanner invocation
-  struct TargetSpecificGlobalCacheState {
+  /// Global cache contents specific to a specific scanner invocation context
+  struct ContextSpecificGlobalCacheState {
     /// All cached module dependencies, in the order in which they were
     /// encountered.
     std::vector<ModuleDependencyID> AllModules;
 
     /// Dependencies for modules that have already been computed.
-    /// This maps a dependency kind to a map of a module's name to a vector of Dependency objects,
-    /// which correspond to instances of the same module that may have been found
-    /// in different sets of search paths.
+    /// This maps a dependency kind to a map of a module's name to a Dependency object
     ModuleDependenciesKindMap ModuleDependenciesMap;
   };
 
@@ -522,23 +513,23 @@ class GlobalModuleDependenciesCache {
 
   /// Dependencies for all Swift source-based modules discovered. Each one is the main
   /// module of a prior invocation of the scanner.
-  llvm::StringMap<ModuleDependencies> SwiftSourceModuleDependenciesMap;
+  ModuleNameToDependencyMap SwiftSourceModuleDependenciesMap;
 
   /// A map from a String representing the target triple of a scanner invocation to the corresponding
   /// cached dependencies discovered so far when using this triple.
-  llvm::StringMap<std::unique_ptr<TargetSpecificGlobalCacheState>> TargetSpecificCacheMap;
+  llvm::StringMap<std::unique_ptr<ContextSpecificGlobalCacheState>> ContextSpecificCacheMap;
 
-  /// The current target triple cache configuration
-  Optional<std::string> CurrentTriple;
+  /// The current context hash configuration
+  Optional<std::string> CurrentContextHash;
 
-  /// The triples used by scanners using this cache, in the order in which they were used
-  std::vector<std::string> AllTriples;
+  /// The context hashes used by scanners using this cache, in the order in which they were used
+  std::vector<std::string> AllContextHashes;
 
   /// Retrieve the dependencies map that corresponds to the given dependency
   /// kind.
-  llvm::StringMap<ModuleDependenciesVector> &
+  ModuleNameToDependencyMap &
   getDependenciesMap(ModuleDependenciesKind kind);
-  const llvm::StringMap<ModuleDependenciesVector> &
+  const ModuleNameToDependencyMap &
   getDependenciesMap(ModuleDependenciesKind kind) const;
 
 public:
@@ -548,47 +539,43 @@ class GlobalModuleDependenciesCache {
   operator=(const GlobalModuleDependenciesCache &) = delete;
   virtual ~GlobalModuleDependenciesCache() {}
 
-  void configureForTriple(std::string triple);
-
-  const std::vector<std::string>& getAllTriples() const {
-    return AllTriples;
-  }
-
 private:
   /// Enforce clients not being allowed to query this cache directly, it must be
   /// wrapped in an instance of `ModuleDependenciesCache`.
   friend class ModuleDependenciesCache;
+  friend class ModuleDependenciesCacheDeserializer;
+  friend class ModuleDependenciesCacheSerializer;
+
+  /// Configure the current state of the cache to respond to queries
+  /// for the specified scanning context hash.
+  void configureForContextHash(std::string scanningContextHash);
+
+  /// Return context hashes of all scanner invocations that have used
+  /// this cache instance.
+  const std::vector<std::string>& getAllContextHashes() const {
+    return AllContextHashes;
+  }
 
   /// Whether we have cached dependency information for the given module.
   bool hasDependencies(StringRef moduleName,
-                       ModuleLookupSpecifics details) const;
+                       Optional<ModuleDependenciesKind> kind) const;
 
-  /// Look for module dependencies for a module with the given name given
-  /// current search paths.
-  ///
-  /// \returns the cached result, or \c None if there is no cached entry.
-  Optional<ModuleDependencies>
-  findDependencies(StringRef moduleName, ModuleLookupSpecifics details) const;
+  /// Return a pointer to the context-specific cache state of the current scanning action.
+  ContextSpecificGlobalCacheState* getCurrentCache() const;
 
-  /// Return a pointer to the target-specific cache state of the current triple configuration.
-  TargetSpecificGlobalCacheState* getCurrentCache() const;
+  /// Return a pointer to the cache state of the specified context hash.
+  ContextSpecificGlobalCacheState* getCacheForScanningContextHash(StringRef scanningContextHash) const;
 
-  /// Return a pointer to the target-specific cache state of the specified triple configuration.
-  TargetSpecificGlobalCacheState* getCacheForTriple(StringRef triple) const;
+  /// Look for source-based module dependency details
+  Optional<ModuleDependencies>
+  findSourceModuleDependency(StringRef moduleName) const;
 
-public:
-  /// Look for module dependencies for a module with the given name.
-  /// This method has a deliberately-obtuse name to indicate that it is not to
-  /// be used for general queries.
+  /// Look for module dependencies for a module with the given name
   ///
   /// \returns the cached result, or \c None if there is no cached entry.
-  Optional<ModuleDependenciesVector>
-  findAllDependenciesIrrespectiveOfSearchPaths(
-      StringRef moduleName, Optional<ModuleDependenciesKind> kind) const;
-
-  /// Look for source-based module dependency details
   Optional<ModuleDependencies>
-  findSourceModuleDependency(StringRef moduleName) const;
+  findDependencies(StringRef moduleName,
+                   Optional<ModuleDependenciesKind> kind) const;
 
   /// Record dependencies for the given module.
   const ModuleDependencies *recordDependencies(StringRef moduleName,
@@ -600,9 +587,9 @@ class GlobalModuleDependenciesCache {
 
   /// Reference the list of all module dependencies that are not source-based modules
   /// (i.e. interface dependencies, binary dependencies, clang dependencies).
-  const std::vector<ModuleDependencyID> &getAllNonSourceModules(StringRef triple) const {
-    auto targetSpecificCache = getCacheForTriple(triple);
-    return targetSpecificCache->AllModules;
+  const std::vector<ModuleDependencyID> &getAllNonSourceModules(StringRef scanningContextHash) const {
+    auto contextSpecificCache = getCacheForScanningContextHash(scanningContextHash);
+    return contextSpecificCache->AllModules;
   }
 
   /// Return the list of all source-based modules discovered by this cache
@@ -616,8 +603,7 @@ class GlobalModuleDependenciesCache {
 /// scanning action, and wraps an instance of a `GlobalModuleDependenciesCache`
 /// which may carry cached scanning information from prior scanning actions.
 /// This cache maintains a store of references to all dependencies found within
-/// the current scanning action (with their values stored in the global Cache),
-/// since these do not require clients to disambiguate them with search paths.
+/// the current scanning action (with their values stored in the global Cache).
 class ModuleDependenciesCache {
 private:
   GlobalModuleDependenciesCache &globalCache;
@@ -628,12 +614,14 @@ class ModuleDependenciesCache {
   ModuleDependenciesKindRefMap ModuleDependenciesMap;
 
   /// Name of the module under scan
-  StringRef mainScanModuleName;
+  std::string mainScanModuleName;
+  /// The context hash of the current scanning invocation
+  std::string scannerContextHash;
+
   /// Set containing all of the Clang modules that have already been seen.
   llvm::StringSet<> alreadySeenClangModules;
   /// The Clang dependency scanner tool
   clang::tooling::dependencies::DependencyScanningTool clangScanningTool;
-
   /// Discovered Clang modules are only cached locally.
   llvm::StringMap<ModuleDependenciesVector> clangModuleDependencies;
 
@@ -646,25 +634,26 @@ class ModuleDependenciesCache {
 
   /// Local cache results lookup, only for modules which were discovered during
   /// the current scanner invocation.
-  bool hasDependencies(StringRef moduleName,
-                       Optional<ModuleDependenciesKind> kind) const;
+  bool hasDependenciesLocalOnly(StringRef moduleName,
+                                Optional<ModuleDependenciesKind> kind) const;
 
   /// Local cache results lookup, only for modules which were discovered during
   /// the current scanner invocation.
   Optional<const ModuleDependencies *>
-  findDependencies(StringRef moduleName,
-                   Optional<ModuleDependenciesKind> kind) const;
+  findDependenciesLocalOnly(StringRef moduleName,
+                            Optional<ModuleDependenciesKind> kind) const;
 
 public:
   ModuleDependenciesCache(GlobalModuleDependenciesCache &globalCache,
-                          StringRef mainScanModuleName);
+                          std::string mainScanModuleName,
+                          std::string scanningContextHash);
   ModuleDependenciesCache(const ModuleDependenciesCache &) = delete;
   ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete;
 
 public:
   /// Whether we have cached dependency information for the given module.
   bool hasDependencies(StringRef moduleName,
-                       ModuleLookupSpecifics details) const;
+                       Optional<ModuleDependenciesKind> kind) const;
 
   /// Produce a reference to the Clang scanner tool associated with this cache
   clang::tooling::dependencies::DependencyScanningTool& getClangScannerTool() {
@@ -674,24 +663,12 @@ class ModuleDependenciesCache {
     return alreadySeenClangModules;
   }
 
-  /// Look for module dependencies for a module with the given name given
-  /// current search paths.
+  /// Look for module dependencies for a module with the given name
   ///
   /// \returns the cached result, or \c None if there is no cached entry.
   Optional<ModuleDependencies>
-  findDependencies(StringRef moduleName, ModuleLookupSpecifics details) const;
-
-  /// Look for module dependencies for a module with the given name.
-  /// This method has a deliberately-obtuse name to indicate that it is not to
-  /// be used for general queries.
-  ///
-  /// \returns the cached result, or \c None if there is no cached entry.
-  Optional<ModuleDependenciesVector>
-  findAllDependenciesIrrespectiveOfSearchPaths(
-      StringRef moduleName, Optional<ModuleDependenciesKind> kind) const {
-    return globalCache.findAllDependenciesIrrespectiveOfSearchPaths(moduleName,
-                                                                    kind);
-  }
+  findDependencies(StringRef moduleName,
+                   Optional<ModuleDependenciesKind> kind) const;
 
   /// Record dependencies for the given module.
   void recordDependencies(StringRef moduleName,

include/swift/AST/SILOptions.h

@@ -281,6 +281,12 @@ class SILOptions {
     return llvm::hash_value(0);
   }
 
+  /// Return a hash code of any components from these options that should
+  /// contribute to a Swift Dependency Scanning hash.
+  llvm::hash_code getModuleScanningHashComponents() const {
+    return llvm::hash_value(0);
+  }
+
   bool shouldOptimize() const {
     return OptMode > OptimizationMode::NoOptimization;
   }

include/swift/AST/SearchPathOptions.h

@@ -421,6 +421,12 @@ class SearchPathOptions {
                                            RuntimeLibraryImportPaths.end()),
                         DisableModulesValidateSystemDependencies);
   }
+
+  /// Return a hash code of any components from these options that should
+  /// contribute to a Swift Dependency Scanning hash.
+  llvm::hash_code getModuleScanningHashComponents() const {
+    return getPCHHashComponents();
+  }
 };
 }
 

include/swift/Basic/DiagnosticOptions.h

@@ -86,6 +86,12 @@ class DiagnosticOptions {
     // Nothing here that contributes anything significant when emitting the PCH.
     return llvm::hash_value(0);
   }
+
+  /// Return a hash code of any components from these options that should
+  /// contribute to a Swift Dependency Scanning hash.
+  llvm::hash_code getModuleScanningHashComponents() const {
+    return llvm::hash_value(0);
+  }
 };
 
 } // end namespace swift

include/swift/Basic/LangOptions.h

@@ -628,6 +628,12 @@ namespace swift {
       return llvm::hash_combine(Target.str(), OS.str());
     }
 
+    /// Return a hash code of any components from these options that should
+    /// contribute to a Swift Dependency Scanning hash.
+    llvm::hash_code getModuleScanningHashComponents() const {
+      return getPCHHashComponents();
+    }
+
   private:
     llvm::SmallVector<std::pair<PlatformConditionKind, std::string>, 6>
         PlatformConditionValues;
@@ -832,6 +838,12 @@ namespace swift {
                           EnableClangSPI);
     }
 
+    /// Return a hash code of any components from these options that should
+    /// contribute to a Swift Dependency Scanning hash.
+    llvm::hash_code getModuleScanningHashComponents() const {
+      return getPCHHashComponents();
+    }
+
     std::vector<std::string> getRemappedExtraArgs(
         std::function<std::string(StringRef)> pathRemapCallback) const;
   };

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -35,9 +35,8 @@ using llvm::BCRecordLayout;
 using llvm::BCVBR;
 
 /// Every .moddepcache file begins with these 4 bytes, for easy identification.
-const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D',
-                                                                  'C'};
-const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 2;
+const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D','C'};
+const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 3;
 /// Increment this on every change.
 const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 0;
 
@@ -56,7 +55,7 @@ using IdentifierIDArryField = llvm::BCArray<IdentifierIDField>;
 
 /// Identifiers used to refer to the above arrays
 using FileIDArrayIDField = IdentifierIDField;
-using TripleIDField = IdentifierIDField;
+using ContextHashIDField = IdentifierIDField;
 using DependencyIDArrayIDField = IdentifierIDField;
 using FlagIDArrayIDField = IdentifierIDField;
 
@@ -118,7 +117,7 @@ using IdentifierArrayLayout =
 using ModuleInfoLayout =
     BCRecordLayout<MODULE_NODE,             // ID
                    IdentifierIDField,       // module name
-                   TripleIDField,           // target triple
+                   ContextHashIDField,      // 
                    DependencyIDArrayIDField // directDependencies
                    >;
 

include/swift/Frontend/Frontend.h

@@ -345,6 +345,11 @@ class CompilerInvocation {
   /// in generating a cached PCH file for the bridging header.
   std::string getPCHHash() const;
 
+  /// Retrieve a module hash string that is suitable for uniquely
+  /// identifying the conditions under which the current module is built,
+  /// from the perspective of a dependency scanning action.
+  std::string getModuleScanningHash() const;
+
   /// Retrieve the stdlib kind to implicitly import.
   ImplicitStdlibKind getImplicitStdlibKind() const {
     if (FrontendOpts.InputMode == FrontendOptions::ParseInputMode::SIL) {

include/swift/Frontend/FrontendOptions.h

@@ -430,6 +430,12 @@ class FrontendOptions {
     return llvm::hash_value(0);
   }
 
+  /// Return a hash code of any components from these options that should
+  /// contribute to a Swift Dependency Scanning hash.
+  llvm::hash_code getModuleScanningHashComponents() const {
+    return llvm::hash_value(0);
+  }
+
   StringRef determineFallbackModuleName() const;
 
   bool isCompilingExactlyOneSwiftFile() const {