[Dependency Scanning] Add support for placing explicitly-built SDK modules into a separate module cache

artemcm · artemcm · commit 88dec5199eb7 · 2025-03-19T09:17:04.000-06:00
With '-sdk-module-cache-path', Swift textual interfaces found in the SDK will be built into a separate SDK-specific module cache.
Clang modules are not yet affected by this change, pending addition of the required API.
diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h
@@ -1149,8 +1149,10 @@ class ModuleDependenciesCache {
   std::string mainScanModuleName;
   /// The context hash of the current scanning invocation
   std::string scannerContextHash;
-  /// The location of where the built modules will be output to
+  /// The location of where the explicitly-built modules will be output to
   std::string moduleOutputPath;
+  /// The location of where the explicitly-built SDK modules will be output to
+  std::string sdkModuleOutputPath;
   /// The timestamp of the beginning of the scanning query action
   /// using this cache
   const llvm::sys::TimePoint<> scanInitializationTime;
@@ -1164,9 +1166,10 @@ class ModuleDependenciesCache {
 
 public:
   ModuleDependenciesCache(SwiftDependencyScanningService &globalScanningService,
-                          std::string mainScanModuleName,
-                          std::string moduleOutputPath,
-                          std::string scanningContextHash);
+                          const std::string &mainScanModuleName,
+                          const std::string &moduleOutputPath,
+                          const std::string &sdkModuleOutputPath,
+                          const std::string &scanningContextHash);
   ModuleDependenciesCache(const ModuleDependenciesCache &) = delete;
   ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete;
 
@@ -1200,6 +1203,7 @@ class ModuleDependenciesCache {
     alreadySeenClangModules.insert(newModule);
   }
   std::string getModuleOutputPath() const { return moduleOutputPath; }
+  std::string getSDKModuleOutputPath() const { return sdkModuleOutputPath; }
 
   /// Query all dependencies
   ModuleDependencyIDSetVector
diff --git a/include/swift/AST/ModuleLoader.h b/include/swift/AST/ModuleLoader.h
@@ -372,7 +372,7 @@ class ModuleLoader {
   /// if no such module exists.
   virtual llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
   getModuleDependencies(Identifier moduleName,
-                        StringRef moduleOutputPath,
+                        StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
                         const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
                         clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
                         InterfaceSubContextDelegate &delegate,
diff --git a/include/swift/ClangImporter/ClangImporter.h b/include/swift/ClangImporter/ClangImporter.h
@@ -488,7 +488,7 @@ class ClangImporter final : public ClangModuleLoader {
       StringRef moduleOutputPath, RemapPathCallback remapPath = nullptr);
 
   llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
-  getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath,
+  getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
                         const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
                         clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
                         InterfaceSubContextDelegate &delegate,
diff --git a/include/swift/DependencyScan/ModuleDependencyScanner.h b/include/swift/DependencyScan/ModuleDependencyScanner.h
@@ -37,17 +37,18 @@ class ModuleDependencyScanningWorker {
 
 private:
   /// Retrieve the module dependencies for the Clang module with the given name.
-  ModuleDependencyVector
-  scanFilesystemForClangModuleDependency(Identifier moduleName,
-                                         StringRef moduleOutputPath,
-                                         const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenModules,
-                                         llvm::PrefixMapper *prefixMapper);
+  ModuleDependencyVector scanFilesystemForClangModuleDependency(
+      Identifier moduleName, StringRef moduleOutputPath,
+      StringRef sdkModuleOutputPath,
+      const llvm::DenseSet<clang::tooling::dependencies::ModuleID>
+          &alreadySeenModules,
+      llvm::PrefixMapper *prefixMapper);
 
   /// Retrieve the module dependencies for the Swift module with the given name.
-  ModuleDependencyVector
-  scanFilesystemForSwiftModuleDependency(Identifier moduleName, StringRef moduleOutputPath,
-                                         llvm::PrefixMapper *prefixMapper,
-                                         bool isTestableImport = false);
+  ModuleDependencyVector scanFilesystemForSwiftModuleDependency(
+      Identifier moduleName, StringRef moduleOutputPath,
+      StringRef sdkModuleOutputPath, llvm::PrefixMapper *prefixMapper,
+      bool isTestableImport = false);
 
   /// Store cache entry for include tree.
   llvm::Error
diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h
@@ -103,6 +103,10 @@ class FrontendOptions {
   /// dependency scanning.
   std::string ExplicitModulesOutputPath;
 
+  /// The path to output explicitly-built SDK module dependencies. Only relevant during
+  /// dependency scanning.
+  std::string ExplicitSDKModulesOutputPath;
+
   /// The path to look in to find backup .swiftinterface files if those found
   /// from SDKs are failing.
   std::string BackupModuleInterfaceDir;
diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td
@@ -2167,6 +2167,10 @@ def clang_scanner_module_cache_path : Separate<["-"], "clang-scanner-module-cach
   Flags<[FrontendOption, DoesNotAffectIncrementalBuild, ArgumentIsPath]>,
   HelpText<"Specifies the Clang dependency scanner module cache path">;
 
+def sdk_module_cache_path : Separate<["-"], "sdk-module-cache-path">,
+  Flags<[FrontendOption, DoesNotAffectIncrementalBuild, ArgumentIsPath]>,
+  HelpText<"Specifies the module cache path for explicitly-built SDK modules">;
+
 def scanner_prefix_map : Separate<["-"], "scanner-prefix-map">,
   Flags<[FrontendOption, NewDriverOnlyOption]>,
   HelpText<"Remap paths reported by dependency scanner">, MetaVarName<"<prefix=replacement>">;
diff --git a/include/swift/Sema/SourceLoader.h b/include/swift/Sema/SourceLoader.h
@@ -99,7 +99,7 @@ class SourceLoader : public ModuleLoader {
   }
 
   llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
-  getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath,
+  getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
                         const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
                         clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
                         InterfaceSubContextDelegate &delegate,
diff --git a/include/swift/Serialization/ScanningLoaders.h b/include/swift/Serialization/ScanningLoaders.h
@@ -39,20 +39,24 @@ class SwiftModuleScanner : public SerializedModuleLoaderBase {
 
   InterfaceSubContextDelegate &astDelegate;
 
-  /// Location where pre-built moduels are to be built into.
+  /// Location where pre-built modules are to be built into.
   std::string moduleOutputPath;
+  /// Location where pre-built SDK modules are to be built into.
+  std::string sdkModuleOutputPath;
 
 public:
   std::optional<ModuleDependencyInfo> dependencies;
 
   SwiftModuleScanner(ASTContext &ctx, ModuleLoadingMode LoadMode,
                      Identifier moduleName,
                      InterfaceSubContextDelegate &astDelegate,
-                     StringRef moduleOutputPath, ScannerKind kind = MDS_plain)
+                     StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
+                     ScannerKind kind = MDS_plain)
       : SerializedModuleLoaderBase(ctx, nullptr, LoadMode,
                                    /*IgnoreSwiftSourceInfoFile=*/true),
         kind(kind), moduleName(moduleName), astDelegate(astDelegate),
-        moduleOutputPath(moduleOutputPath) {}
+        moduleOutputPath(moduleOutputPath),
+        sdkModuleOutputPath(sdkModuleOutputPath) {}
 
   std::error_code findModuleFilesInDirectory(
       ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
@@ -93,9 +97,11 @@ class PlaceholderSwiftModuleScanner : public SwiftModuleScanner {
                                 Identifier moduleName,
                                 StringRef PlaceholderDependencyModuleMap,
                                 InterfaceSubContextDelegate &astDelegate,
-                                StringRef moduleOutputPath)
+                                StringRef moduleOutputPath,
+                                StringRef sdkModuleOutputPath)
       : SwiftModuleScanner(ctx, LoadMode, moduleName, astDelegate,
-                           moduleOutputPath, MDS_placeholder) {
+                           moduleOutputPath, sdkModuleOutputPath,
+                           MDS_placeholder) {
 
     // FIXME: Find a better place for this map to live, to avoid
     // doing the parsing on every module.
diff --git a/include/swift/Serialization/SerializedModuleLoader.h b/include/swift/Serialization/SerializedModuleLoader.h
@@ -264,7 +264,7 @@ class SerializedModuleLoaderBase : public ModuleLoader {
   virtual void verifyAllModules() override;
 
   virtual llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
-  getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath,
+  getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
                         const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
                         clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
                         InterfaceSubContextDelegate &delegate,
diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp
@@ -765,12 +765,13 @@ bool SwiftDependencyScanningService::setupCachingDependencyScanningService(
 
 ModuleDependenciesCache::ModuleDependenciesCache(
     SwiftDependencyScanningService &globalScanningService,
-    std::string mainScanModuleName, std::string moduleOutputPath,
-    std::string scannerContextHash)
+    const std::string &mainScanModuleName, const std::string &moduleOutputPath,
+    const std::string &sdkModuleOutputPath, const std::string &scannerContextHash)
     : globalScanningService(globalScanningService),
       mainScanModuleName(mainScanModuleName),
       scannerContextHash(scannerContextHash),
       moduleOutputPath(moduleOutputPath),
+      sdkModuleOutputPath(sdkModuleOutputPath),
       scanInitializationTime(std::chrono::system_clock::now()) {
   for (auto kind = ModuleDependencyKind::FirstKind;
        kind != ModuleDependencyKind::LastKind; ++kind)
diff --git a/lib/ClangImporter/ClangModuleDependencyScanner.cpp b/lib/ClangImporter/ClangModuleDependencyScanner.cpp
@@ -403,6 +403,7 @@ computeClangWorkingDirectory(const std::vector<std::string> &commandLineArgs,
 ModuleDependencyVector
 ClangImporter::getModuleDependencies(Identifier moduleName,
                                      StringRef moduleOutputPath,
+                                     StringRef sdkModuleOutputPath,
                                      const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
                                      clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
                                      InterfaceSubContextDelegate &delegate,
@@ -419,10 +420,11 @@ ClangImporter::getModuleDependencies(Identifier moduleName,
     return {};
   }
   std::string workingDir = *optionalWorkingDir;
-
   auto lookupModuleOutput =
       [moduleOutputPath](const ModuleDeps &MD,
                          ModuleOutputKind MOK) -> std::string {
+    // ACTODO: Once the clang scanner gets the required functionality,
+    // use sdkModuleOutputPath for modules whose modulemap is a part of the SDK.
     return moduleCacheRelativeLookupModuleOutput(MD.ID, MOK, moduleOutputPath);
   };
 
diff --git a/lib/DependencyScan/DependencyScanningTool.cpp b/lib/DependencyScan/DependencyScanningTool.cpp
@@ -286,6 +286,7 @@ DependencyScanningTool::getDependencies(
   ModuleDependenciesCache cache(
       *ScanningService, QueryContext.ScanInstance->getMainModule()->getNameStr().str(),
       QueryContext.ScanInstance->getInvocation().getFrontendOptions().ExplicitModulesOutputPath,
+      QueryContext.ScanInstance->getInvocation().getFrontendOptions().ExplicitSDKModulesOutputPath,
       QueryContext.ScanInstance->getInvocation().getModuleScanningHash());
   // Execute the scanning action, retrieving the in-memory result
   auto DependenciesOrErr = performModuleScan(*QueryContext.ScanInstance.get(), 
@@ -316,6 +317,7 @@ DependencyScanningTool::getImports(ArrayRef<const char *> Command,
   ModuleDependenciesCache cache(
       *ScanningService, QueryContext.ScanInstance->getMainModule()->getNameStr().str(),
       QueryContext.ScanInstance->getInvocation().getFrontendOptions().ExplicitModulesOutputPath,
+      QueryContext.ScanInstance->getInvocation().getFrontendOptions().ExplicitSDKModulesOutputPath,
       QueryContext.ScanInstance->getInvocation().getModuleScanningHash());
   auto DependenciesOrErr = performModulePrescan(*QueryContext.ScanInstance.get(), 
                                                 QueryContext.ScanDiagnostics.get(),
diff --git a/lib/DependencyScan/ModuleDependencyScanner.cpp b/lib/DependencyScan/ModuleDependencyScanner.cpp
@@ -240,21 +240,22 @@ ModuleDependencyScanningWorker::ModuleDependencyScanningWorker(
 ModuleDependencyVector
 ModuleDependencyScanningWorker::scanFilesystemForSwiftModuleDependency(
     Identifier moduleName, StringRef moduleOutputPath,
-    llvm::PrefixMapper *prefixMapper, bool isTestableImport) {
+    StringRef sdkModuleOutputPath, llvm::PrefixMapper *prefixMapper,
+    bool isTestableImport) {
   return swiftScannerModuleLoader->getModuleDependencies(
-      moduleName, moduleOutputPath,
+      moduleName, moduleOutputPath, sdkModuleOutputPath,
       {}, clangScanningTool, *scanningASTDelegate,
       prefixMapper, isTestableImport);
 }
 
 ModuleDependencyVector
 ModuleDependencyScanningWorker::scanFilesystemForClangModuleDependency(
-    Identifier moduleName,
-    StringRef moduleOutputPath,
+    Identifier moduleName, StringRef moduleOutputPath,
+    StringRef sdkModuleOutputPath,
     const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenModules,
     llvm::PrefixMapper *prefixMapper) {
   return clangScannerModuleLoader->getModuleDependencies(
-      moduleName, moduleOutputPath,
+      moduleName, moduleOutputPath, sdkModuleOutputPath,
       alreadySeenModules, clangScanningTool,
       *scanningASTDelegate, prefixMapper, false);
 }
@@ -519,6 +520,7 @@ ModuleDependencyScanner::getNamedClangModuleDependencyInfo(
       [&cache, moduleIdentifier](ModuleDependencyScanningWorker *ScanningWorker) {
         return ScanningWorker->scanFilesystemForClangModuleDependency(
           moduleIdentifier, cache.getModuleOutputPath(),
+          cache.getSDKModuleOutputPath(),
           cache.getAlreadySeenClangModules(),
           cache.getScanService().getPrefixMapper());
       });
@@ -557,6 +559,7 @@ ModuleDependencyScanner::getNamedSwiftModuleDependencyInfo(
       [&cache, moduleIdentifier](ModuleDependencyScanningWorker *ScanningWorker) {
         return ScanningWorker->scanFilesystemForSwiftModuleDependency(
           moduleIdentifier, cache.getModuleOutputPath(),
+          cache.getSDKModuleOutputPath(),
           cache.getScanService().getPrefixMapper());
       });
   if (moduleDependencies.empty())
@@ -911,7 +914,8 @@ ModuleDependencyScanner::resolveAllClangModuleDependencies(
              moduleIdentifier](ModuleDependencyScanningWorker *ScanningWorker) {
               return ScanningWorker->scanFilesystemForClangModuleDependency(
                   moduleIdentifier, cache.getModuleOutputPath(),
-                  seenClangModules, cache.getScanService().getPrefixMapper());
+                  cache.getSDKModuleOutputPath(), seenClangModules,
+                  cache.getScanService().getPrefixMapper());
             });
 
         // Update the `moduleLookupResult` and cache all discovered dependencies
@@ -1100,6 +1104,7 @@ void ModuleDependencyScanner::resolveSwiftImportsForModule(
              isTestable](ModuleDependencyScanningWorker *ScanningWorker) {
               return ScanningWorker->scanFilesystemForSwiftModuleDependency(
                   moduleIdentifier, cache.getModuleOutputPath(),
+                  cache.getSDKModuleOutputPath(),
                   cache.getScanService().getPrefixMapper(), isTestable);
             });
 
@@ -1288,6 +1293,7 @@ void ModuleDependencyScanner::resolveSwiftOverlayDependenciesForModule(
          moduleIdentifier](ModuleDependencyScanningWorker *ScanningWorker) {
           return ScanningWorker->scanFilesystemForSwiftModuleDependency(
               moduleIdentifier, cache.getModuleOutputPath(),
+              cache.getSDKModuleOutputPath(),
               cache.getScanService().getPrefixMapper());
         });
     swiftOverlayLookupResult.insert_or_assign(moduleName, moduleDependencies);
diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp
@@ -1168,6 +1168,7 @@ bool swift::dependencies::scanDependencies(CompilerInstance &CI) {
   ModuleDependenciesCache cache(
       *service, CI.getMainModule()->getNameStr().str(),
       CI.getInvocation().getFrontendOptions().ExplicitModulesOutputPath,
+      CI.getInvocation().getFrontendOptions().ExplicitSDKModulesOutputPath,
       CI.getInvocation().getModuleScanningHash());
 
   if (service->setupCachingDependencyScanningService(CI))
@@ -1203,6 +1204,7 @@ bool swift::dependencies::prescanDependencies(CompilerInstance &instance) {
   ModuleDependenciesCache cache(
       *singleUseService, instance.getMainModule()->getNameStr().str(),
       instance.getInvocation().getFrontendOptions().ExplicitModulesOutputPath,
+      instance.getInvocation().getFrontendOptions().ExplicitSDKModulesOutputPath,
       instance.getInvocation().getModuleScanningHash());
 
   // Execute import prescan, and write JSON output to the output stream
diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp
@@ -81,6 +81,11 @@ bool ArgsToFrontendOptionsConverter::convert(
     clang::driver::Driver::getDefaultModuleCachePath(defaultPath);
     Opts.ExplicitModulesOutputPath = defaultPath.str().str();
   }
+  if (const Arg *A = Args.getLastArg(OPT_sdk_module_cache_path)) {
+    Opts.ExplicitSDKModulesOutputPath = A->getValue();
+  } else {
+    Opts.ExplicitSDKModulesOutputPath = Opts.ExplicitModulesOutputPath;
+  }
   if (const Arg *A = Args.getLastArg(OPT_backup_module_interface_path)) {
     Opts.BackupModuleInterfaceDir = A->getValue();
   }
diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp
@@ -866,7 +866,8 @@ bool CompilerInstance::setUpModuleLoaders() {
         std::make_unique<PlaceholderSwiftModuleScanner>(
             *Context, MLM, mainModuleName,
             Context->SearchPathOpts.PlaceholderDependencyModuleMap, ASTDelegate,
-            getInvocation().getFrontendOptions().ExplicitModulesOutputPath);
+            getInvocation().getFrontendOptions().ExplicitModulesOutputPath,
+            getInvocation().getFrontendOptions().ExplicitSDKModulesOutputPath);
     Context->addModuleLoader(std::move(PSMS));
   }
 
diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp
@@ -2890,6 +2890,18 @@ void setOutputPath(ResultTy &resolvedOutputPath, const StringRef &moduleName,
                    const CompilerInvocation &CI, const ArgListTy &extraArgs) {
   auto &outputPath = resolvedOutputPath.outputPath;
   outputPath = CI.getClangModuleCachePath();
+ 
+  // Dependency-scanner-specific module output path handling
+  if ((CI.getFrontendOptions().RequestedAction ==
+       FrontendOptions::ActionType::ScanDependencies)) {
+    if (!sdkPath.empty() &&
+        hasPrefix(llvm::sys::path::begin(interfacePath.str()), llvm::sys::path::end(interfacePath.str()),
+                  llvm::sys::path::begin(sdkPath), llvm::sys::path::end(sdkPath)))
+      outputPath = CI.getFrontendOptions().ExplicitSDKModulesOutputPath;
+    else
+      outputPath = CI.getFrontendOptions().ExplicitModulesOutputPath;
+  }
+
   llvm::sys::path::append(outputPath, moduleName);
   outputPath.append("-");
   auto hashStart = outputPath.size();
diff --git a/lib/Sema/SourceLoader.cpp b/lib/Sema/SourceLoader.cpp
@@ -156,7 +156,7 @@ void SourceLoader::loadExtensions(NominalTypeDecl *nominal,
 
 ModuleDependencyVector
 SourceLoader::getModuleDependencies(Identifier moduleName,
-                                    StringRef moduleOutputPath,
+                                    StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
                                     const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
                                     clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
                                     InterfaceSubContextDelegate &delegate,
diff --git a/lib/Serialization/ScanningLoaders.cpp b/lib/Serialization/ScanningLoaders.cpp
diff --git a/test/ScanDependencies/separate_sdk_module_cache.swift b/test/ScanDependencies/separate_sdk_module_cache.swift

Original file line number	Diff line number	Diff line change
`@@ -99,7 +99,7 @@ class SourceLoader : public ModuleLoader {`
`99`	`99`	`}`
`100`	`100`
`101`	`101`	`llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>`
`102`		`- getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath,`
	`102`	`+ getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath, StringRef sdkModuleOutputPath,`
`103`	`103`	`const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,`
`104`	`104`	`clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,`
`105`	`105`	`InterfaceSubContextDelegate &delegate,`