Skip to content

Commit d443fdd

Browse files
rastogishubhamrastogishubham
authored
Merge pull request #74202 from rastogishubham/MCCASReplay
Add replay support for MCCAS in Swift.
2 parents f38b78f + 8dabf58 commit d443fdd

File tree

9 files changed

+191
-7
lines changed

9 files changed

+191
-7
lines changed

‎include/swift/AST/DiagnosticsCommon.def‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ ERROR(error_opening_output,none,
3232
ERROR(error_closing_output,none,
3333
"error closing '%0' for output: %1", (StringRef, StringRef))
3434

35+
ERROR(error_mccas,none,
36+
"error trying to materialize MCCAS object file: %0", (StringRef))
37+
3538
ERROR(cannot_find_group_info_file,none,
3639
"cannot find group info file at path: '%0'", (StringRef))
3740

‎include/swift/Frontend/CASOutputBackends.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,10 @@ class SwiftCASOutputBackend : public llvm::vfs::OutputBackend {
4949
llvm::Error storeCachedDiagnostics(unsigned InputIndex,
5050
llvm::StringRef Bytes);
5151

52+
/// Store the MCCAS CASID \p ID as the object file output for the input
53+
/// that corresponds to the \p OutputFilename
54+
llvm::Error storeMCCASObjectID(StringRef OutputFilename, llvm::cas::CASID ID);
55+
5256
private:
5357
class Implementation;
5458
Implementation &Impl;

‎include/swift/Frontend/CachingUtils.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ bool replayCachedCompilerOutputs(
4444
llvm::cas::ObjectStore &CAS, llvm::cas::ActionCache &Cache,
4545
llvm::cas::ObjectRef BaseKey, DiagnosticEngine &Diag,
4646
const FrontendInputsAndOutputs &InputsAndOutputs,
47-
CachingDiagnosticsProcessor &CDP, bool CacheRemarks);
47+
CachingDiagnosticsProcessor &CDP, bool CacheRemarks, bool UseCASBackend);
4848

4949
/// Load the cached compile result from cache.
5050
std::unique_ptr<llvm::MemoryBuffer> loadCachedCompileResultFromCacheKey(

‎lib/Frontend/CASOutputBackends.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,21 @@ Error SwiftCASOutputBackend::storeCachedDiagnostics(unsigned InputIndex,
142142
file_types::ID::TY_CachedDiagnostics);
143143
}
144144

145+
Error SwiftCASOutputBackend::storeMCCASObjectID(StringRef OutputFilename,
146+
llvm::cas::CASID ID) {
147+
auto Input = Impl.OutputToInputMap.find(OutputFilename);
148+
if (Input == Impl.OutputToInputMap.end())
149+
return llvm::createStringError("InputIndex for output file not found!");
150+
auto InputIndex = Input->second.first;
151+
auto MCRef = Impl.CAS.getReference(ID);
152+
if (!MCRef)
153+
return createStringError("Invalid CASID: " + ID.toString() +
154+
". No associated ObjectRef found!");
155+
156+
Impl.OutputRefs[InputIndex].insert({file_types::TY_Object, *MCRef});
157+
return Impl.finalizeCacheKeysFor(InputIndex);
158+
}
159+
145160
void SwiftCASOutputBackend::Implementation::initBackend(
146161
const FrontendInputsAndOutputs &InputsAndOutputs) {
147162
// FIXME: The output to input map might not be enough for example all the

‎lib/Frontend/CachingUtils.cpp‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
#include "llvm/CAS/HierarchicalTreeBuilder.h"
3131
#include "llvm/CAS/ObjectStore.h"
3232
#include "llvm/CAS/TreeEntry.h"
33+
#include "llvm/MCCAS/MCCASObjectV1.h"
3334
#include "llvm/Option/ArgList.h"
3435
#include "llvm/Option/OptTable.h"
3536
#include "llvm/Support/Debug.h"
@@ -135,7 +136,7 @@ lookupCacheKey(ObjectStore &CAS, ActionCache &Cache, ObjectRef CacheKey) {
135136
bool replayCachedCompilerOutputs(
136137
ObjectStore &CAS, ActionCache &Cache, ObjectRef BaseKey,
137138
DiagnosticEngine &Diag, const FrontendInputsAndOutputs &InputsAndOutputs,
138-
CachingDiagnosticsProcessor &CDP, bool CacheRemarks) {
139+
CachingDiagnosticsProcessor &CDP, bool CacheRemarks, bool UseCASBackend) {
139140
bool CanReplayAllOutput = true;
140141
struct OutputEntry {
141142
std::string Path;
@@ -144,6 +145,7 @@ bool replayCachedCompilerOutputs(
144145
};
145146
SmallVector<OutputEntry> OutputProxies;
146147
std::optional<OutputEntry> DiagnosticsOutput;
148+
std::string ObjFile;
147149

148150
auto replayOutputsForInputFile = [&](const std::string &InputPath,
149151
unsigned InputIndex,
@@ -177,7 +179,6 @@ bool replayCachedCompilerOutputs(
177179
CachedResultLoader Loader(CAS, **OutputRef);
178180
LLVM_DEBUG(llvm::dbgs() << "DEBUG: lookup cache key \'" << OutID.toString()
179181
<< "\' for input \'" << InputPath << "\n";);
180-
181182
if (auto Err = Loader.replay([&](file_types::ID Kind,
182183
ObjectRef Ref) -> Error {
183184
auto OutputPath = Outputs.find(Kind);
@@ -189,6 +190,9 @@ bool replayCachedCompilerOutputs(
189190
if (!Proxy)
190191
return Proxy.takeError();
191192

193+
if (Kind == file_types::ID::TY_Object && UseCASBackend)
194+
ObjFile = OutputPath->second;
195+
192196
if (Kind == file_types::ID::TY_CachedDiagnostics) {
193197
assert(!DiagnosticsOutput && "more than 1 diagnotics found");
194198
DiagnosticsOutput = OutputEntry{OutputPath->second, OutID, *Proxy};
@@ -282,7 +286,14 @@ bool replayCachedCompilerOutputs(
282286
toString(File.takeError()));
283287
continue;
284288
}
285-
*File << Output.Proxy.getData();
289+
290+
if (UseCASBackend && Output.Path == ObjFile) {
291+
auto Schema = std::make_unique<llvm::mccasformats::v1::MCSchema>(CAS);
292+
if (auto E = Schema->serializeObjectFile(Output.Proxy, *File))
293+
Diag.diagnose(SourceLoc(), diag::error_mccas, toString(std::move(E)));
294+
} else
295+
*File << Output.Proxy.getData();
296+
286297
if (auto E = File->keep()) {
287298
Diag.diagnose(SourceLoc(), diag::error_closing_output, Output.Path,
288299
toString(std::move(E)));

‎lib/Frontend/Frontend.cpp‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -475,10 +475,31 @@ void CompilerInstance::setupOutputBackend() {
475475

476476
// Mirror the output into CAS.
477477
if (supportCaching()) {
478+
auto &InAndOuts = Invocation.getFrontendOptions().InputsAndOutputs;
478479
CASOutputBackend = createSwiftCachingOutputBackend(
479-
*CAS, *ResultCache, *CompileJobBaseKey,
480-
Invocation.getFrontendOptions().InputsAndOutputs,
480+
*CAS, *ResultCache, *CompileJobBaseKey, InAndOuts,
481481
Invocation.getFrontendOptions().RequestedAction);
482+
483+
if (Invocation.getIRGenOptions().UseCASBackend) {
484+
auto OutputFiles = InAndOuts.copyOutputFilenames();
485+
std::unordered_set<std::string> OutputFileSet(
486+
std::make_move_iterator(OutputFiles.begin()),
487+
std::make_move_iterator(OutputFiles.end()));
488+
// Filter the object file output if MCCAS is enabled, we do not want to
489+
// store the object file itself, but store the MCCAS CASID instead.
490+
auto FilterBackend = llvm::vfs::makeFilteringOutputBackend(
491+
CASOutputBackend,
492+
[&, OutputFileSet](StringRef Path,
493+
std::optional<llvm::vfs::OutputConfig> Config) {
494+
if (InAndOuts.getPrincipalOutputType() != file_types::ID::TY_Object)
495+
return true;
496+
return !(OutputFileSet.find(Path.str()) != OutputFileSet.end());
497+
});
498+
OutputBackend =
499+
llvm::vfs::makeMirroringOutputBackend(OutputBackend, FilterBackend);
500+
return;
501+
}
502+
482503
OutputBackend =
483504
llvm::vfs::makeMirroringOutputBackend(OutputBackend, CASOutputBackend);
484505
}

‎lib/FrontendTool/FrontendTool.cpp‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1327,7 +1327,8 @@ static bool tryReplayCompilerResults(CompilerInstance &Instance) {
13271327
Instance.getObjectStore(), Instance.getActionCache(),
13281328
*Instance.getCompilerBaseKey(), Instance.getDiags(),
13291329
Instance.getInvocation().getFrontendOptions().InputsAndOutputs, *CDP,
1330-
Instance.getInvocation().getCASOptions().EnableCachingRemarks);
1330+
Instance.getInvocation().getCASOptions().EnableCachingRemarks,
1331+
Instance.getInvocation().getIRGenOptions().UseCASBackend);
13311332

13321333
// If we didn't replay successfully, re-start capture.
13331334
if (!replayed)
@@ -1584,6 +1585,18 @@ static bool generateCode(CompilerInstance &Instance, StringRef OutputFilename,
15841585
createTargetMachine(opts, Instance.getASTContext());
15851586

15861587
TargetMachine->Options.MCOptions.CAS = Instance.getSharedCASInstance();
1588+
1589+
if (Instance.getInvocation().getCASOptions().EnableCaching &&
1590+
opts.UseCASBackend)
1591+
TargetMachine->Options.MCOptions.ResultCallBack =
1592+
[&](const llvm::cas::CASID &ID) -> llvm::Error {
1593+
if (auto Err = Instance.getCASOutputBackend().storeMCCASObjectID(
1594+
OutputFilename, ID))
1595+
return Err;
1596+
1597+
return llvm::Error::success();
1598+
};
1599+
15871600
// Free up some compiler resources now that we have an IRModule.
15881601
freeASTContextIfPossible(Instance);
15891602

‎test/CAS/cache_replay_mccas.swift‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
// REQUIRES: OS=macosx
2+
// RUN: %empty-directory(%t)
3+
// RUN: split-file %s %t
4+
5+
// RUN: %target-swift-frontend -scan-dependencies -module-name Test -O -module-cache-path %t/clang-module-cache \
6+
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \
7+
// RUN: %t/test.swift -I %t -o %t/deps.json -cache-compile-job -cas-backend -cas-backend-mode=verify -cas-path %t/cas
8+
9+
/// Check clang module
10+
// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:Dummy > %t/dummy.cmd
11+
// RUN: %swift_frontend_plain @%t/dummy.cmd -Rcache-compile-job 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
12+
// RUN: %swift_frontend_plain @%t/dummy.cmd -Rcache-compile-job 2>&1 | %FileCheck --check-prefix=CACHE-HIT-CLANG %s
13+
14+
// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json
15+
// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid
16+
17+
// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd
18+
// RUN: echo "\"-disable-implicit-string-processing-module-import\"" >> %t/MyApp.cmd
19+
// RUN: echo "\"-disable-implicit-concurrency-module-import\"" >> %t/MyApp.cmd
20+
// RUN: echo "\"-disable-implicit-swift-modules\"" >> %t/MyApp.cmd
21+
// RUN: echo "\"-parse-stdlib\"" >> %t/MyApp.cmd
22+
// RUN: echo "\"-explicit-swift-module-map-file\"" >> %t/MyApp.cmd
23+
// RUN: echo "\"@%t/map.casid\"" >> %t/MyApp.cmd
24+
25+
/// Run the command first time, expect cache miss.
26+
// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c -emit-dependencies \
27+
// RUN: -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
28+
// RUN: test -f %t/test.o
29+
// RUN: test -f %t/test.d
30+
// RUN: test -f %t/Test.swiftmodule
31+
32+
/// Expect cache hit for second time.
33+
// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c -emit-dependencies \
34+
// RUN: -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s
35+
// RUN: test -f %t/test.o
36+
// RUN: test -f %t/test.d
37+
// RUN: test -f %t/Test.swiftmodule
38+
39+
/// Expect cache miss a subset of outputs.
40+
// RUN %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c \
41+
// RUN -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
42+
43+
/// Cache hit for retry.
44+
// RUN %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c \
45+
// RUN -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s
46+
47+
/// Skip cache
48+
// RUN %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job -cache-disable-replay %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c \
49+
// RUN -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --allow-empty --check-prefix=SKIP-CACHE %s
50+
51+
// CACHE-MISS: remark: cache miss for input
52+
// CACHE-HIT: remark: replay output file '<cached-diagnostics>': key 'llvmcas://{{.*}}'
53+
// CACHE-HIT: remark: replay output file '{{.*}}{{/|\\}}test.o': key 'llvmcas://{{.*}}'
54+
// CACHE-HIT: remark: replay output file '{{.*}}{{/|\\}}Test.swiftmodule': key 'llvmcas://{{.*}}'
55+
// CACHE-HIT-CLANG: remark: replay output file '<cached-diagnostics>': key 'llvmcas://{{.*}}'
56+
// CACHE-HIT-CLANG: remark: replay output file '{{.*}}{{/|\\}}Dummy-{{.*}}.pcm': key 'llvmcas://{{.*}}'
57+
// SKIP-CACHE-NOT: remark:
58+
59+
//--- test.swift
60+
import Dummy
61+
func testFunc() {}
62+
63+
//--- module.modulemap
64+
module Dummy {
65+
umbrella header "Dummy.h"
66+
}
67+
68+
//--- Dummy.h
69+
void dummy(void);

‎test/CAS/cache_replay_multiple_files_mccas.swift‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// REQUIRES: OS=macosx
2+
// RUN: %empty-directory(%t)
3+
// RUN: split-file %s %t
4+
5+
// RUN: %target-swift-frontend -scan-dependencies -module-name Test -O \
6+
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \
7+
// RUN: %t/test.swift %t/foo.swift -o %t/deps.json -cache-compile-job -cas-backend -cas-backend-mode=verify -cas-path %t/cas
8+
9+
// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json
10+
// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid
11+
12+
// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd
13+
// RUN: echo "\"-disable-implicit-string-processing-module-import\"" >> %t/MyApp.cmd
14+
// RUN: echo "\"-disable-implicit-concurrency-module-import\"" >> %t/MyApp.cmd
15+
// RUN: echo "\"-parse-stdlib\"" >> %t/MyApp.cmd
16+
17+
/// Test compile multiple inputs with batch mode.
18+
// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift %t/foo.swift -emit-module -o %t/Test.swiftmodule \
19+
// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
20+
// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job -primary-file %t/test.swift %t/foo.swift -c -o %t/test.o \
21+
// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
22+
// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -primary-file %t/foo.swift -c -o %t/foo.o \
23+
// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
24+
// RUN: test -f %t/test.o
25+
// RUN: test -f %t/foo.o
26+
// RUN: test -f %t/Test.swiftmodule
27+
28+
/// Expect cache hit second time
29+
// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift %t/foo.swift -emit-module -o %t/Test.swiftmodule \
30+
// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s
31+
// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job -primary-file %t/test.swift %t/foo.swift -c -o %t/test.o \
32+
// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s
33+
// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -primary-file %t/foo.swift -c -o %t/foo.o \
34+
// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s
35+
// RUN: test -f %t/test.o
36+
// RUN: test -f %t/foo.o
37+
// RUN: test -f %t/Test.swiftmodule
38+
39+
//--- test.swift
40+
func testFunc() {}
41+
42+
//--- foo.swift
43+
func foo() {}
44+
45+
// CACHE-MISS: remark: cache miss for input
46+
// CACHE-MISS-NOT: remark: replay output file
47+
// CACHE-HIT: remark: replay output file
48+
// CACHE-HIT-NOT: remark: cache miss for input

0 commit comments

Comments
 (0)