Skip to content

Commit 74883b3

Browse files
artemcmartemcm
authored
Merge pull request #78075 from artemcm/ModuleCacheSerializeUpToSpeed
[Dependency Scanning] Restore functionality of a binary serialization format of the dependency scanner state/cache
2 parents d7d85bb + 7049c5b commit 74883b3

15 files changed

+1175
-590
lines changed

‎include/swift/AST/DiagnosticsCommon.def‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,9 @@ WARNING(warn_scanner_deserialize_failed, none,
218218
REMARK(remark_reuse_cache, none,
219219
"Re-using serialized module scanning dependency cache from: '%0'.", (StringRef))
220220

221+
REMARK(remark_scanner_uncached_lookups, none,
222+
"Module Dependency Scanner queries executed: '%0'.", (unsigned))
223+
221224
REMARK(remark_save_cache, none,
222225
"Serializing module scanning dependency cache to: '%0'.", (StringRef))
223226

‎include/swift/AST/ModuleDependencies.h‎

Lines changed: 36 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -176,13 +176,6 @@ class ModuleDependencyInfoStorageBase {
176176
public:
177177
const ModuleDependencyKind dependencyKind;
178178

179-
ModuleDependencyInfoStorageBase(ModuleDependencyKind dependencyKind,
180-
ArrayRef<LinkLibrary> linkLibraries,
181-
StringRef moduleCacheKey = "")
182-
: dependencyKind(dependencyKind), linkLibraries(linkLibraries),
183-
moduleCacheKey(moduleCacheKey.str()),
184-
finalized(false) {}
185-
186179
ModuleDependencyInfoStorageBase(
187180
ModuleDependencyKind dependencyKind,
188181
ArrayRef<ScannerImportStatementInfo> moduleImports,
@@ -303,11 +296,14 @@ class SwiftInterfaceModuleDependenciesStorage
303296
SwiftInterfaceModuleDependenciesStorage(
304297
StringRef moduleOutputPath, StringRef swiftInterfaceFile,
305298
ArrayRef<StringRef> compiledModuleCandidates,
299+
ArrayRef<ScannerImportStatementInfo> moduleImports,
300+
ArrayRef<ScannerImportStatementInfo> optionalModuleImports,
306301
ArrayRef<StringRef> buildCommandLine, ArrayRef<LinkLibrary> linkLibraries,
307302
ArrayRef<StringRef> extraPCMArgs, StringRef contextHash, bool isFramework,
308303
bool isStatic, StringRef RootID, StringRef moduleCacheKey,
309304
StringRef userModuleVersion)
310305
: ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftInterface,
306+
moduleImports, optionalModuleImports,
311307
linkLibraries, moduleCacheKey),
312308
moduleOutputPath(moduleOutputPath),
313309
swiftInterfaceFile(swiftInterfaceFile),
@@ -350,9 +346,12 @@ class SwiftSourceModuleDependenciesStorage
350346

351347
SwiftSourceModuleDependenciesStorage(
352348
StringRef RootID, ArrayRef<StringRef> buildCommandLine,
349+
ArrayRef<ScannerImportStatementInfo> moduleImports,
350+
ArrayRef<ScannerImportStatementInfo> optionalModuleImports,
353351
ArrayRef<StringRef> bridgingHeaderBuildCommandLine,
354352
ArrayRef<StringRef> extraPCMArgs)
355-
: ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftSource, {}),
353+
: ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftSource,
354+
moduleImports, optionalModuleImports, {}),
356355
textualModuleDetails(extraPCMArgs, buildCommandLine, RootID),
357356
testableImports(llvm::StringSet<>()),
358357
bridgingHeaderBuildCommandLine(bridgingHeaderBuildCommandLine.begin(),
@@ -500,6 +499,7 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase {
500499
StringRef clangIncludeTreeRoot,
501500
StringRef moduleCacheKey, bool IsSystem)
502501
: ModuleDependencyInfoStorageBase(ModuleDependencyKind::Clang,
502+
{}, {},
503503
linkLibraries, moduleCacheKey),
504504
pcmOutputPath(pcmOutputPath), mappedPCMPath(mappedPCMPath),
505505
moduleMapFile(moduleMapFile), contextHash(contextHash),
@@ -531,7 +531,7 @@ class SwiftPlaceholderModuleDependencyStorage
531531
SwiftPlaceholderModuleDependencyStorage(StringRef compiledModulePath,
532532
StringRef moduleDocPath,
533533
StringRef sourceInfoPath)
534-
: ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftPlaceholder,
534+
: ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftPlaceholder, {}, {},
535535
{}),
536536
compiledModulePath(compiledModulePath), moduleDocPath(moduleDocPath),
537537
sourceInfoPath(sourceInfoPath) {}
@@ -586,13 +586,16 @@ class ModuleDependencyInfo {
586586
static ModuleDependencyInfo forSwiftInterfaceModule(
587587
StringRef moduleOutputPath, StringRef swiftInterfaceFile,
588588
ArrayRef<StringRef> compiledCandidates, ArrayRef<StringRef> buildCommands,
589+
ArrayRef<ScannerImportStatementInfo> moduleImports,
590+
ArrayRef<ScannerImportStatementInfo> optionalModuleImports,
589591
ArrayRef<LinkLibrary> linkLibraries, ArrayRef<StringRef> extraPCMArgs,
590592
StringRef contextHash, bool isFramework, bool isStatic,
591593
StringRef CASFileSystemRootID, StringRef moduleCacheKey,
592594
StringRef userModuleVersion) {
593595
return ModuleDependencyInfo(
594596
std::make_unique<SwiftInterfaceModuleDependenciesStorage>(
595597
moduleOutputPath, swiftInterfaceFile, compiledCandidates,
598+
moduleImports, optionalModuleImports,
596599
buildCommands, linkLibraries, extraPCMArgs, contextHash,
597600
isFramework, isStatic, CASFileSystemRootID, moduleCacheKey,
598601
userModuleVersion));
@@ -619,14 +622,28 @@ class ModuleDependencyInfo {
619622
static ModuleDependencyInfo
620623
forSwiftSourceModule(const std::string &CASFileSystemRootID,
621624
ArrayRef<StringRef> buildCommands,
625+
ArrayRef<ScannerImportStatementInfo> moduleImports,
626+
ArrayRef<ScannerImportStatementInfo> optionalModuleImports,
622627
ArrayRef<StringRef> bridgingHeaderBuildCommands,
623628
ArrayRef<StringRef> extraPCMArgs) {
624629
return ModuleDependencyInfo(
625630
std::make_unique<SwiftSourceModuleDependenciesStorage>(
626-
CASFileSystemRootID, buildCommands, bridgingHeaderBuildCommands,
631+
CASFileSystemRootID, buildCommands, moduleImports,
632+
optionalModuleImports, bridgingHeaderBuildCommands,
627633
extraPCMArgs));
628634
}
629635

636+
static ModuleDependencyInfo
637+
forSwiftSourceModule() {
638+
return ModuleDependencyInfo(
639+
std::make_unique<SwiftSourceModuleDependenciesStorage>(
640+
StringRef(), ArrayRef<StringRef>(),
641+
ArrayRef<ScannerImportStatementInfo>(),
642+
ArrayRef<ScannerImportStatementInfo>(),
643+
ArrayRef<StringRef>(),
644+
ArrayRef<StringRef>()));
645+
}
646+
630647
/// Describe the module dependencies for a Clang module that can be
631648
/// built from a module map and headers.
632649
static ModuleDependencyInfo forClangModule(
@@ -769,6 +786,15 @@ class ModuleDependencyInfo {
769786
setLinkLibraries(const ArrayRef<LinkLibrary> linkLibraries) {
770787
storage->linkLibraries.assign(linkLibraries.begin(), linkLibraries.end());
771788
}
789+
790+
const ArrayRef<std::string> getAuxiliaryFiles() const {
791+
return storage->auxiliaryFiles;
792+
}
793+
794+
void
795+
setAuxiliaryFiles(const ArrayRef<std::string> auxiliaryFiles) {
796+
storage->auxiliaryFiles.assign(auxiliaryFiles.begin(), auxiliaryFiles.end());
797+
}
772798

773799
bool isStaticLibrary() const {
774800
if (auto *detail = getAsSwiftInterfaceModule())

‎include/swift/DependencyScan/DependencyScanningTool.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,6 @@ class DependencyScanningTool {
108108
const llvm::StringSet<> &PlaceholderModules,
109109
StringRef WorkingDirectory);
110110

111-
/// Discard the tool's current `SharedCache` and start anew.
112-
void resetCache();
113111
/// Query diagnostics consumed so far.
114112
std::vector<DependencyScanDiagnosticCollector::ScannerDiagnosticInfo> getDiagnostics();
115113
/// Discared the collection of diagnostics encountered so far.

‎include/swift/DependencyScan/ModuleDependencyScanner.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,9 @@ class ModuleDependencyScanner {
9595
getNamedSwiftModuleDependencyInfo(StringRef moduleName,
9696
ModuleDependenciesCache &cache);
9797

98+
/// How many filesystem lookups were performed by the scanner
99+
unsigned getNumLookups() { return NumLookups; }
100+
98101
private:
99102
/// Main routine that computes imported module dependency transitive
100103
/// closure for the given module.
@@ -166,6 +169,8 @@ class ModuleDependencyScanner {
166169
llvm::StdThreadPool ScanningThreadPool;
167170
/// Protect worker access.
168171
std::mutex WorkersLock;
172+
/// Count of filesystem queries performed
173+
std::atomic<unsigned> NumLookups = 0;
169174
};
170175

171176
} // namespace swift

‎include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h‎

Lines changed: 76 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,9 @@ using llvm::BCVBR;
3939

4040
/// Every .moddepcache file begins with these 4 bytes, for easy identification.
4141
const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D','C'};
42-
const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR =
43-
7; // isSystem
42+
const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 8;
4443
/// Increment this on every change.
45-
const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 1;
44+
const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 0;
4645

4746
/// Various identifiers in this format will rely on having their strings mapped
4847
/// using this ID.
@@ -57,6 +56,14 @@ using IsFrameworkField = BCFixed<1>;
5756
using IsSystemField = BCFixed<1>;
5857
/// A bit that indicates whether or not a module is that of a static archive
5958
using IsStaticField = BCFixed<1>;
59+
/// A bit taht indicates whether or not a link library is a force-load one
60+
using IsForceLoadField = BCFixed<1>;
61+
/// A bit taht indicates whether or not an import statement is optional
62+
using IsOptionalImport = BCFixed<1>;
63+
64+
/// Source location fields
65+
using LineNumberField = BCFixed<32>;
66+
using ColumnNumberField = BCFixed<32>;
6067

6168
/// Arrays of various identifiers, distinguished for readability
6269
using IdentifierIDArryField = llvm::BCArray<IdentifierIDField>;
@@ -65,9 +72,14 @@ using ModuleIDArryField = llvm::BCArray<IdentifierIDField>;
6572
/// Identifiers used to refer to the above arrays
6673
using FileIDArrayIDField = IdentifierIDField;
6774
using ContextHashIDField = IdentifierIDField;
75+
using ModuleCacheKeyIDField = IdentifierIDField;
6876
using ImportArrayIDField = IdentifierIDField;
77+
using LinkLibrariesArrayIDField = IdentifierIDField;
78+
using MacroDependenciesArrayIDField = IdentifierIDField;
6979
using FlagIDArrayIDField = IdentifierIDField;
7080
using DependencyIDArrayIDField = IdentifierIDField;
81+
using AuxiliaryFilesArrayIDField = IdentifierIDField;
82+
using SourceLocationIDArrayIDField = IdentifierIDField;
7183

7284
/// The ID of the top-level block containing the dependency graph
7385
const unsigned GRAPH_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID;
@@ -76,11 +88,20 @@ const unsigned GRAPH_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID;
7688
/// zero or more IDENTIFIER records that contain various strings seen in the graph
7789
/// (e.g. file names or compiler flags), followed by zero or more IDENTIFIER_ARRAY records
7890
/// which are arrays of identifiers seen in the graph (e.g. list of source files or list of compile flags),
91+
/// followed by zero or more LINK_LIBRARY_NODE records along with associated
92+
///
7993
/// followed by zero or more MODULE_NODE, *_DETAILS_NODE pairs of records.
8094
namespace graph_block {
8195
enum {
8296
METADATA = 1,
8397
MODULE_NODE,
98+
LINK_LIBRARY_NODE,
99+
LINK_LIBRARY_ARRAY_NODE,
100+
MACRO_DEPENDENCY_NODE,
101+
MACRO_DEPENDENCY_ARRAY_NODE,
102+
IMPORT_STATEMENT_NODE,
103+
IMPORT_STATEMENT_ARRAY_NODE,
104+
OPTIONAL_IMPORT_STATEMENT_ARRAY_NODE,
84105
SWIFT_INTERFACE_MODULE_DETAILS_NODE,
85106
SWIFT_SOURCE_MODULE_DETAILS_NODE,
86107
SWIFT_PLACEHOLDER_MODULE_DETAILS_NODE,
@@ -95,7 +116,7 @@ using MetadataLayout = BCRecordLayout<
95116
METADATA, // ID
96117
BCFixed<16>, // Inter-Module Dependency graph format major version
97118
BCFixed<16>, // Inter-Module Dependency graph format minor version
98-
BCBlob // Compiler version string
119+
BCBlob // Scanner Invocation Context Hash
99120
>;
100121

101122
// After the metadata record, we have zero or more identifier records,
@@ -117,6 +138,43 @@ using IdentifierNodeLayout = BCRecordLayout<IDENTIFIER_NODE, BCBlob>;
117138
using IdentifierArrayLayout =
118139
BCRecordLayout<IDENTIFIER_ARRAY_NODE, IdentifierIDArryField>;
119140

141+
// ACTODO: Comment
142+
using LinkLibraryLayout =
143+
BCRecordLayout<LINK_LIBRARY_NODE, // ID
144+
IdentifierIDField, // libraryName
145+
IsFrameworkField, // isFramework
146+
IsForceLoadField // forceLoad
147+
>;
148+
// ACTODO: Comment
149+
using LinkLibraryArrayLayout =
150+
BCRecordLayout<LINK_LIBRARY_ARRAY_NODE, IdentifierIDArryField>;
151+
152+
// ACTODO: Comment
153+
using MacroDependencyLayout =
154+
BCRecordLayout<MACRO_DEPENDENCY_NODE, // ID
155+
IdentifierIDField, // macroModuleName
156+
IdentifierIDField, // libraryPath
157+
IdentifierIDField // executablePath
158+
>;
159+
// ACTODO: Comment
160+
using MacroDependencyArrayLayout =
161+
BCRecordLayout<MACRO_DEPENDENCY_ARRAY_NODE, IdentifierIDArryField>;
162+
163+
// ACTODO: Comment
164+
using ImportStatementLayout =
165+
BCRecordLayout<IMPORT_STATEMENT_NODE, // ID
166+
IdentifierIDField, // importIdentifier
167+
IdentifierIDField, // bufferIdentifier
168+
LineNumberField, // lineNumber
169+
ColumnNumberField, // columnNumber
170+
IsOptionalImport // isOptional
171+
>;
172+
// ACTODO: Comment
173+
using ImportStatementArrayLayout =
174+
BCRecordLayout<IMPORT_STATEMENT_ARRAY_NODE, IdentifierIDArryField>;
175+
using OptionalImportStatementArrayLayout =
176+
BCRecordLayout<OPTIONAL_IMPORT_STATEMENT_ARRAY_NODE, IdentifierIDArryField>;
177+
120178
// After the array records, we have a sequence of Module info
121179
// records, each of which is followed by one of:
122180
// - SwiftInterfaceModuleDetails
@@ -125,12 +183,18 @@ using IdentifierArrayLayout =
125183
// - SwiftPlaceholderModuleDetails
126184
// - ClangModuleDetails
127185
using ModuleInfoLayout =
128-
BCRecordLayout<MODULE_NODE, // ID
129-
IdentifierIDField, // moduleName
130-
ContextHashIDField, // contextHash
131-
ImportArrayIDField, // moduleImports
132-
ImportArrayIDField, // optionalModuleImports
133-
DependencyIDArrayIDField // resolvedDirectModuleDependencies
186+
BCRecordLayout<MODULE_NODE, // ID
187+
IdentifierIDField, // moduleName
188+
ImportArrayIDField, // imports
189+
ImportArrayIDField, // optionalImports
190+
LinkLibrariesArrayIDField, // linkLibraries
191+
MacroDependenciesArrayIDField, // macroDependencies
192+
DependencyIDArrayIDField, // importedSwiftModules
193+
DependencyIDArrayIDField, // importedClangModules
194+
DependencyIDArrayIDField, // crossImportOverlayModules
195+
DependencyIDArrayIDField, // swiftOverlayDependencies
196+
ModuleCacheKeyIDField, // moduleCacheKey
197+
AuxiliaryFilesArrayIDField // auxiliaryFiles
134198
>;
135199

136200
using SwiftInterfaceModuleDetailsLayout =
@@ -147,7 +211,6 @@ using SwiftInterfaceModuleDetailsLayout =
147211
FileIDArrayIDField, // sourceFiles
148212
FileIDArrayIDField, // bridgingSourceFiles
149213
IdentifierIDField, // bridgingModuleDependencies
150-
DependencyIDArrayIDField, // swiftOverlayDependencies
151214
IdentifierIDField, // CASFileSystemRootID
152215
IdentifierIDField, // bridgingHeaderIncludeTree
153216
IdentifierIDField, // moduleCacheKey
@@ -161,7 +224,6 @@ using SwiftSourceModuleDetailsLayout =
161224
FileIDArrayIDField, // sourceFiles
162225
FileIDArrayIDField, // bridgingSourceFiles
163226
FileIDArrayIDField, // bridgingModuleDependencies
164-
DependencyIDArrayIDField, // swiftOverlayDependencies
165227
IdentifierIDField, // CASFileSystemRootID
166228
IdentifierIDField, // bridgingHeaderIncludeTree
167229
FlagIDArrayIDField, // buildCommandLine
@@ -173,8 +235,8 @@ using SwiftBinaryModuleDetailsLayout =
173235
FileIDField, // compiledModulePath
174236
FileIDField, // moduleDocPath
175237
FileIDField, // moduleSourceInfoPath
176-
DependencyIDArrayIDField, // swiftOverlayDependencies
177238
FileIDField, // headerImport
239+
FileIDField, // definingInterfacePath
178240
IdentifierIDField, // headerModuleDependencies
179241
FileIDArrayIDField, // headerSourceFiles
180242
IsFrameworkField, // isFramework
@@ -220,7 +282,7 @@ bool readInterModuleDependenciesCache(llvm::StringRef path,
220282
/// Returns true if there was an error.
221283
bool writeInterModuleDependenciesCache(DiagnosticEngine &diags,
222284
llvm::vfs::OutputBackend &backend,
223-
llvm::StringRef path,
285+
llvm::StringRef outputPath,
224286
const ModuleDependenciesCache &cache);
225287

226288
/// Tries to write out the given dependency cache with the given

‎lib/AST/ModuleDependencies.cpp‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -44,16 +44,17 @@ bool ModuleDependencyInfo::isTextualSwiftModule() const {
4444
return isSwiftInterfaceModule() || isSwiftSourceModule();
4545
}
4646

47-
ModuleDependencyKind &operator++(ModuleDependencyKind &e) {
48-
if (e == ModuleDependencyKind::LastKind) {
49-
llvm_unreachable(
50-
"Attempting to increment last enum value on ModuleDependencyKind");
47+
namespace {
48+
ModuleDependencyKind &operator++(ModuleDependencyKind &e) {
49+
if (e == ModuleDependencyKind::LastKind) {
50+
llvm_unreachable(
51+
"Attempting to increment last enum value on ModuleDependencyKind");
52+
}
53+
e = ModuleDependencyKind(
54+
static_cast<std::underlying_type<ModuleDependencyKind>::type>(e) + 1);
55+
return e;
5156
}
52-
e = ModuleDependencyKind(
53-
static_cast<std::underlying_type<ModuleDependencyKind>::type>(e) + 1);
54-
return e;
5557
}
56-
5758
bool ModuleDependencyInfo::isSwiftInterfaceModule() const {
5859
return isa<SwiftInterfaceModuleDependenciesStorage>(storage.get());
5960
}
@@ -783,7 +784,7 @@ ModuleDependenciesCache::findDependency(
783784
auto dep = optionalDep.value();
784785
if (dep->getAsSwiftSourceModule() &&
785786
moduleName != mainScanModuleName &&
786-
moduleName != "DummyMainModuleForResolvingCrossImportOverlays") {
787+
moduleName != "MainModuleCrossImportOverlays") {
787788
return std::nullopt;
788789
}
789790
}

‎lib/DependencyScan/DependencyScanningTool.cpp‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -356,11 +356,6 @@ DependencyScanningTool::getDependencies(
356356
return BatchScanResults;
357357
}
358358

359-
void DependencyScanningTool::resetCache() {
360-
llvm::sys::SmartScopedLock<true> Lock(DependencyScanningToolStateLock);
361-
ScanningService.reset(new SwiftDependencyScanningService());
362-
}
363-
364359
std::vector<
365360
DependencyScanDiagnosticCollector::ScannerDiagnosticInfo>
366361
DependencyScanningTool::getDiagnostics() {

0 commit comments

Comments
 (0)