Skip to content

Commit cdd1ac9

Browse files
cachemeifyoucancachemeifyoucan
committed
[Caching] Support CrossImport modules for caching build
Add support for cross import modules by ingesting swiftoverlay files for the cross import into CAS file system. The long-term better fix will be just passing the cross import information from scanner to swift-frontend so frontend doesn't need to read overlay files again to figure out the cross import module. rdar://123839248
1 parent 6e18ffe commit cdd1ac9

File tree

8 files changed

+176
-83
lines changed

8 files changed

+176
-83
lines changed

‎include/swift/AST/ModuleDependencies.h‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,10 @@ class ModuleDependencyInfoStorageBase {
172172
/// The cache key for the produced module.
173173
std::string moduleCacheKey;
174174

175+
/// Auxiliary files that help to construct other dependencies (e.g.
176+
/// command-line), no need to be saved to reconstruct from cache.
177+
std::vector<std::string> auxiliaryFiles;
178+
175179
/// The direct dependency of the module is resolved by scanner.
176180
bool resolved;
177181
/// ModuleDependencyInfo is finalized (with all transitive dependencies
@@ -661,6 +665,24 @@ class ModuleDependencyInfo {
661665
llvm_unreachable("Unexpected type");
662666
}
663667

668+
void addAuxiliaryFile(const std::string &file) {
669+
storage->auxiliaryFiles.emplace_back(file);
670+
}
671+
672+
void updateCASFileSystemRootID(const std::string &rootID) {
673+
if (isSwiftInterfaceModule())
674+
cast<SwiftInterfaceModuleDependenciesStorage>(storage.get())
675+
->textualModuleDetails.CASFileSystemRootID = rootID;
676+
else if (isSwiftSourceModule())
677+
cast<SwiftSourceModuleDependenciesStorage>(storage.get())
678+
->textualModuleDetails.CASFileSystemRootID = rootID;
679+
else if (isClangModule())
680+
cast<ClangModuleDependencyStorage>(storage.get())->CASFileSystemRootID =
681+
rootID;
682+
else
683+
llvm_unreachable("Unexpected type");
684+
}
685+
664686
bool isResolved() const {
665687
return storage->resolved;
666688
}
@@ -790,7 +812,8 @@ class ModuleDependencyInfo {
790812
/// Collect a map from a secondary module name to a list of cross-import
791813
/// overlays, when this current module serves as the primary module.
792814
llvm::StringMap<llvm::SmallSetVector<Identifier, 4>>
793-
collectCrossImportOverlayNames(ASTContext &ctx, StringRef moduleName) const;
815+
collectCrossImportOverlayNames(ASTContext &ctx, StringRef moduleName,
816+
std::vector<std::string> &overlayFiles) const;
794817
};
795818

796819
using ModuleDependencyVector = llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>;

‎include/swift/DependencyScan/ModuleDependencyScanner.h‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,8 @@ class ModuleDependencyScanner {
7272
DiagnosticEngine &diags, bool ParallelScan);
7373

7474
/// Identify the scanner invocation's main module's dependencies
75-
llvm::ErrorOr<ModuleDependencyInfo> getMainModuleDependencyInfo(
76-
ModuleDecl *mainModule,
77-
std::optional<SwiftDependencyTracker> tracker = std::nullopt);
75+
llvm::ErrorOr<ModuleDependencyInfo>
76+
getMainModuleDependencyInfo(ModuleDecl *mainModule);
7877

7978
/// Resolve module dependencies of the given module, computing a full
8079
/// transitive closure dependency graph.

‎include/swift/Serialization/ScanningLoaders.h‎

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -41,20 +41,17 @@ class SwiftModuleScanner : public SerializedModuleLoaderBase {
4141
/// Location where pre-built moduels are to be built into.
4242
std::string moduleOutputPath;
4343

44-
std::optional<SwiftDependencyTracker> dependencyTracker;
45-
4644
public:
4745
std::optional<ModuleDependencyInfo> dependencies;
4846

49-
SwiftModuleScanner(
50-
ASTContext &ctx, ModuleLoadingMode LoadMode, Identifier moduleName,
51-
InterfaceSubContextDelegate &astDelegate, StringRef moduleOutputPath,
52-
ScannerKind kind = MDS_plain,
53-
std::optional<SwiftDependencyTracker> tracker = std::nullopt)
47+
SwiftModuleScanner(ASTContext &ctx, ModuleLoadingMode LoadMode,
48+
Identifier moduleName,
49+
InterfaceSubContextDelegate &astDelegate,
50+
StringRef moduleOutputPath, ScannerKind kind = MDS_plain)
5451
: SerializedModuleLoaderBase(ctx, nullptr, LoadMode,
5552
/*IgnoreSwiftSourceInfoFile=*/true),
5653
kind(kind), moduleName(moduleName), astDelegate(astDelegate),
57-
moduleOutputPath(moduleOutputPath), dependencyTracker(tracker) {}
54+
moduleOutputPath(moduleOutputPath) {}
5855

5956
std::error_code findModuleFilesInDirectory(
6057
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
@@ -91,13 +88,13 @@ class PlaceholderSwiftModuleScanner : public SwiftModuleScanner {
9188
llvm::BumpPtrAllocator Allocator;
9289

9390
public:
94-
PlaceholderSwiftModuleScanner(
95-
ASTContext &ctx, ModuleLoadingMode LoadMode, Identifier moduleName,
96-
StringRef PlaceholderDependencyModuleMap,
97-
InterfaceSubContextDelegate &astDelegate, StringRef moduleOutputPath,
98-
std::optional<SwiftDependencyTracker> tracker = std::nullopt)
91+
PlaceholderSwiftModuleScanner(ASTContext &ctx, ModuleLoadingMode LoadMode,
92+
Identifier moduleName,
93+
StringRef PlaceholderDependencyModuleMap,
94+
InterfaceSubContextDelegate &astDelegate,
95+
StringRef moduleOutputPath)
9996
: SwiftModuleScanner(ctx, LoadMode, moduleName, astDelegate,
100-
moduleOutputPath, MDS_placeholder, tracker) {
97+
moduleOutputPath, MDS_placeholder) {
10198

10299
// FIXME: Find a better place for this map to live, to avoid
103100
// doing the parsing on every module.

‎lib/AST/ModuleLoader.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,8 +186,9 @@ void ModuleLoader::findOverlayFiles(SourceLoc diagLoc, ModuleDecl *module,
186186
}
187187

188188
llvm::StringMap<llvm::SmallSetVector<Identifier, 4>>
189-
ModuleDependencyInfo::collectCrossImportOverlayNames(ASTContext &ctx,
190-
StringRef moduleName) const {
189+
ModuleDependencyInfo::collectCrossImportOverlayNames(
190+
ASTContext &ctx, StringRef moduleName,
191+
std::vector<std::string> &overlayFiles) const {
191192
using namespace llvm::sys;
192193
using namespace file_types;
193194
std::optional<std::string> modulePath;
@@ -239,6 +240,7 @@ ModuleDependencyInfo::collectCrossImportOverlayNames(ASTContext &ctx,
239240
ModuleDecl::collectCrossImportOverlay(ctx, file, moduleName,
240241
bystandingModule);
241242
result[bystandingModule] = std::move(overlayNames);
243+
overlayFiles.push_back(file.str());
242244
});
243245
return result;
244246
}

‎lib/DependencyScan/ModuleDependencyScanner.cpp‎

Lines changed: 20 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -326,8 +326,7 @@ static void findAllImportedClangModules(StringRef moduleName,
326326
}
327327

328328
llvm::ErrorOr<ModuleDependencyInfo>
329-
ModuleDependencyScanner::getMainModuleDependencyInfo(
330-
ModuleDecl *mainModule, std::optional<SwiftDependencyTracker> tracker) {
329+
ModuleDependencyScanner::getMainModuleDependencyInfo(ModuleDecl *mainModule) {
331330
// Main module file name.
332331
auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile);
333332
llvm::SmallString<32> mainModulePath = mainModule->getName().str();
@@ -345,35 +344,9 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(
345344
ExtraPCMArgs.begin(),
346345
{"-Xcc", "-target", "-Xcc", ScanASTContext.LangOpts.Target.str()});
347346

348-
std::string rootID;
349-
std::vector<std::string> buildArgs;
350347
auto clangImporter =
351348
static_cast<ClangImporter *>(ScanASTContext.getClangModuleLoader());
352-
if (tracker) {
353-
tracker->startTracking();
354-
for (auto fileUnit : mainModule->getFiles()) {
355-
auto sf = dyn_cast<SourceFile>(fileUnit);
356-
if (!sf)
357-
continue;
358-
tracker->trackFile(sf->getFilename());
359-
}
360-
tracker->addCommonSearchPathDeps(
361-
ScanCompilerInvocation.getSearchPathOptions());
362-
// Fetch some dependency files from clang importer.
363-
std::vector<std::string> clangDependencyFiles;
364-
clangImporter->addClangInvovcationDependencies(clangDependencyFiles);
365-
llvm::for_each(clangDependencyFiles,
366-
[&](std::string &file) { tracker->trackFile(file); });
367-
368-
auto root = tracker->createTreeFromDependencies();
369-
if (!root) {
370-
Diagnostics.diagnose(SourceLoc(), diag::error_cas,
371-
toString(root.takeError()));
372-
return std::make_error_code(std::errc::io_error);
373-
}
374-
rootID = root->getID().toString();
375-
}
376-
349+
std::vector<std::string> buildArgs;
377350
if (ScanASTContext.ClangImporterOpts.ClangImporterDirectCC1Scan) {
378351
buildArgs.push_back("-direct-clang-cc1-module-build");
379352
for (auto &arg : clangImporter->getSwiftExplicitModuleDirectCC1Args()) {
@@ -389,7 +362,15 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(
389362
});
390363

391364
auto mainDependencies = ModuleDependencyInfo::forSwiftSourceModule(
392-
rootID, buildCommands, {}, ExtraPCMArgs);
365+
{}, buildCommands, {}, ExtraPCMArgs);
366+
367+
if (ScanASTContext.CASOpts.EnableCaching) {
368+
std::vector<std::string> clangDependencyFiles;
369+
clangImporter->addClangInvovcationDependencies(clangDependencyFiles);
370+
llvm::for_each(clangDependencyFiles, [&](std::string &file) {
371+
mainDependencies.addAuxiliaryFile(file);
372+
});
373+
}
393374

394375
llvm::StringSet<> alreadyAddedModules;
395376
// Compute Implicit dependencies of the main module
@@ -808,6 +789,7 @@ void ModuleDependencyScanner::discoverCrossImportOverlayDependencies(
808789
llvm::function_ref<void(ModuleDependencyID)> action) {
809790
// Modules explicitly imported. Only these can be secondary module.
810791
llvm::SetVector<Identifier> newOverlays;
792+
std::vector<std::string> overlayFiles;
811793
for (auto dep : allDependencies) {
812794
auto moduleName = dep.ModuleName;
813795
// Do not look for overlays of main module under scan
@@ -817,7 +799,7 @@ void ModuleDependencyScanner::discoverCrossImportOverlayDependencies(
817799
auto dependencies = cache.findDependency(moduleName, dep.Kind).value();
818800
// Collect a map from secondary module name to cross-import overlay names.
819801
auto overlayMap = dependencies->collectCrossImportOverlayNames(
820-
ScanASTContext, moduleName);
802+
ScanASTContext, moduleName, overlayFiles);
821803
if (overlayMap.empty())
822804
continue;
823805
for (const auto &dependencyId : allDependencies) {
@@ -879,11 +861,9 @@ void ModuleDependencyScanner::discoverCrossImportOverlayDependencies(
879861

880862
// Update main module's dependencies to include these new overlays.
881863
auto resolvedDummyDep =
882-
*(cache.findDependency(dummyMainName, ModuleDependencyKind::SwiftSource)
883-
.value());
864+
**cache.findDependency(dummyMainName, ModuleDependencyKind::SwiftSource);
884865
auto mainDep =
885-
*(cache.findDependency(mainModuleName, ModuleDependencyKind::SwiftSource)
886-
.value());
866+
**cache.findDependency(mainModuleName, ModuleDependencyKind::SwiftSource);
887867
auto newOverlayDeps = resolvedDummyDep.getDirectModuleDependencies();
888868
auto existingMainDeps = mainDep.getDirectModuleDependencies();
889869
ModuleDependencyIDSet existingMainDepsSet(existingMainDeps.begin(),
@@ -895,6 +875,11 @@ void ModuleDependencyScanner::discoverCrossImportOverlayDependencies(
895875
if (!existingMainDepsSet.count(crossImportOverlayModID))
896876
mainDep.addModuleDependency(crossImportOverlayModID);
897877
});
878+
879+
llvm::for_each(overlayFiles, [&mainDep](const std::string &file) {
880+
mainDep.addAuxiliaryFile(file);
881+
});
882+
898883
cache.updateDependency(
899884
{mainModuleName.str(), ModuleDependencyKind::SwiftSource}, mainDep);
900885

‎lib/DependencyScan/ScanDependencies.cpp‎

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -207,13 +207,11 @@ static llvm::Error resolveExplicitModuleInputs(
207207
auto &service = cache.getScanService();
208208
auto remapPath = [&](StringRef path) { return service.remapPath(path); };
209209
std::vector<std::string> rootIDs;
210-
if (auto ID = resolvingDepInfo.getCASFSRootID())
211-
rootIDs.push_back(*ID);
212-
213210
std::vector<std::string> includeTrees;
214211
if (auto ID = resolvingDepInfo.getClangIncludeTree())
215212
includeTrees.push_back(*ID);
216213

214+
auto tracker = cache.getScanService().createSwiftDependencyTracker();
217215
auto addBridgingHeaderDeps =
218216
[&](const ModuleDependencyInfo &depInfo) -> llvm::Error {
219217
auto sourceDepDetails = depInfo.getAsSwiftSourceModule();
@@ -223,8 +221,7 @@ static llvm::Error resolveExplicitModuleInputs(
223221
if (sourceDepDetails->textualModuleDetails
224222
.CASBridgingHeaderIncludeTreeRootID.empty()) {
225223
if (!sourceDepDetails->textualModuleDetails.bridgingSourceFiles.empty()) {
226-
if (auto tracker =
227-
cache.getScanService().createSwiftDependencyTracker()) {
224+
if (tracker) {
228225
tracker->startTracking();
229226
for (auto &file :
230227
sourceDepDetails->textualModuleDetails.bridgingSourceFiles)
@@ -329,6 +326,41 @@ static llvm::Error resolveExplicitModuleInputs(
329326
auto &CASFS = cache.getScanService().getSharedCachingFS();
330327
auto &CAS = CASFS.getCAS();
331328

329+
assert(tracker && "no caching tracker is available");
330+
// Compute the CASFS root ID for the resolving dependency.
331+
if (auto *sourceDep = resolvingDepInfo.getAsSwiftSourceModule()) {
332+
tracker->startTracking();
333+
tracker->addCommonSearchPathDeps(
334+
instance.getInvocation().getSearchPathOptions());
335+
llvm::for_each(
336+
sourceDep->sourceFiles,
337+
[&tracker](const std::string &file) { tracker->trackFile(file); });
338+
llvm::for_each(
339+
sourceDep->auxiliaryFiles,
340+
[&tracker](const std::string &file) { tracker->trackFile(file); });
341+
auto root = tracker->createTreeFromDependencies();
342+
if (!root)
343+
return root.takeError();
344+
auto rootID = root->getID().toString();
345+
dependencyInfoCopy.updateCASFileSystemRootID(rootID);
346+
rootIDs.push_back(rootID);
347+
} else if (auto *textualDep =
348+
resolvingDepInfo.getAsSwiftInterfaceModule()) {
349+
tracker->startTracking();
350+
tracker->addCommonSearchPathDeps(
351+
instance.getInvocation().getSearchPathOptions());
352+
tracker->trackFile(textualDep->swiftInterfaceFile);
353+
llvm::for_each(
354+
textualDep->auxiliaryFiles,
355+
[&tracker](const std::string &file) { tracker->trackFile(file); });
356+
auto root = tracker->createTreeFromDependencies();
357+
if (!root)
358+
return root.takeError();
359+
auto rootID = root->getID().toString();
360+
dependencyInfoCopy.updateCASFileSystemRootID(rootID);
361+
rootIDs.push_back(rootID);
362+
}
363+
332364
// Update build command line.
333365
if (resolvingDepInfo.isSwiftInterfaceModule() ||
334366
resolvingDepInfo.isSwiftSourceModule()) {
@@ -1898,9 +1930,8 @@ swift::dependencies::performModuleScan(CompilerInstance &instance,
18981930

18991931
// Identify imports of the main module and add an entry for it
19001932
// to the dependency graph.
1901-
auto mainModuleDepInfo = scanner.getMainModuleDependencyInfo(
1902-
instance.getMainModule(),
1903-
cache.getScanService().createSwiftDependencyTracker());
1933+
auto mainModuleDepInfo =
1934+
scanner.getMainModuleDependencyInfo(instance.getMainModule());
19041935
auto mainModuleName = instance.getMainModule()->getNameStr();
19051936
auto mainModuleID = ModuleDependencyID{mainModuleName.str(),
19061937
ModuleDependencyKind::SwiftSource};

‎lib/Serialization/ScanningLoaders.cpp‎

Lines changed: 9 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -205,29 +205,21 @@ SwiftModuleScanner::scanInterfaceFile(Twine moduleInterfacePath,
205205
*moduleDecl, SourceFileKind::Interface, bufferID, parsingOpts);
206206
moduleDecl->addAuxiliaryFile(*sourceFile);
207207

208-
std::string RootID;
209-
if (dependencyTracker) {
210-
dependencyTracker->startTracking();
211-
dependencyTracker->addCommonSearchPathDeps(Ctx.SearchPathOpts);
208+
std::vector<StringRef> ArgsRefs(Args.begin(), Args.end());
209+
Result = ModuleDependencyInfo::forSwiftInterfaceModule(
210+
outputPathBase.str().str(), InPath, compiledCandidates, ArgsRefs,
211+
PCMArgs, Hash, isFramework, {}, /*module-cache-key*/ "");
212+
213+
if (Ctx.CASOpts.EnableCaching) {
212214
std::vector<std::string> clangDependencyFiles;
213215
auto clangImporter =
214216
static_cast<ClangImporter *>(Ctx.getClangModuleLoader());
215217
clangImporter->addClangInvovcationDependencies(clangDependencyFiles);
216218
llvm::for_each(clangDependencyFiles, [&](std::string &file) {
217-
dependencyTracker->trackFile(file);
219+
Result->addAuxiliaryFile(file);
218220
});
219-
dependencyTracker->trackFile(moduleInterfacePath);
220-
auto RootOrError = dependencyTracker->createTreeFromDependencies();
221-
if (!RootOrError)
222-
return llvm::errorToErrorCode(RootOrError.takeError());
223-
RootID = RootOrError->getID().toString();
224221
}
225222

226-
std::vector<StringRef> ArgsRefs(Args.begin(), Args.end());
227-
Result = ModuleDependencyInfo::forSwiftInterfaceModule(
228-
outputPathBase.str().str(), InPath, compiledCandidates, ArgsRefs,
229-
PCMArgs, Hash, isFramework, RootID, /*module-cache-key*/ "");
230-
231223
// Walk the source file to find the import declarations.
232224
llvm::StringSet<> alreadyAddedModules;
233225
Result->addModuleImport(*sourceFile, alreadyAddedModules);
@@ -260,9 +252,6 @@ ModuleDependencyVector SerializedModuleLoaderBase::getModuleDependencies(
260252
ImportPath::Module::Builder builder(moduleName);
261253
auto modulePath = builder.get();
262254
auto moduleId = modulePath.front().Item;
263-
std::optional<SwiftDependencyTracker> tracker = std::nullopt;
264-
if (CacheFS)
265-
tracker = SwiftDependencyTracker(*CacheFS, mapper);
266255

267256
// Do not load interface module if it is testable import.
268257
ModuleLoadingMode MLM =
@@ -277,10 +266,10 @@ ModuleDependencyVector SerializedModuleLoaderBase::getModuleDependencies(
277266
// FIXME: submodules?
278267
scanners.push_back(std::make_unique<PlaceholderSwiftModuleScanner>(
279268
Ctx, MLM, moduleId, Ctx.SearchPathOpts.PlaceholderDependencyModuleMap,
280-
delegate, moduleOutputPath, tracker));
269+
delegate, moduleOutputPath));
281270
scanners.push_back(std::make_unique<SwiftModuleScanner>(
282271
Ctx, MLM, moduleId, delegate, moduleOutputPath,
283-
SwiftModuleScanner::MDS_plain, tracker));
272+
SwiftModuleScanner::MDS_plain));
284273

285274
// Check whether there is a module with this name that we can import.
286275
assert(isa<PlaceholderSwiftModuleScanner>(scanners[0].get()) &&

0 commit comments

Comments
 (0)