Add support for MCCAS in replay logic for swift

rastogishubham · rastogishubham · commit 8dabf5899312 · 2024-07-10T10:19:10.000-07:00
This patch adds support for MCCAS when a cache hit is encountered when
trying to replay a compilation, and uses the MCCAS serialization code
to materialize the object file that is the main output of the
compilation.
diff --git a/include/swift/AST/DiagnosticsCommon.def b/include/swift/AST/DiagnosticsCommon.def
@@ -32,6 +32,9 @@ ERROR(error_opening_output,none,
 ERROR(error_closing_output,none,
       "error closing '%0' for output: %1", (StringRef, StringRef))
 
+ERROR(error_mccas,none,
+      "error trying to materialize MCCAS object file: %0", (StringRef))
+
 ERROR(cannot_find_group_info_file,none,
       "cannot find group info file at path: '%0'", (StringRef))
 
diff --git a/include/swift/Frontend/CachingUtils.h b/include/swift/Frontend/CachingUtils.h
@@ -44,7 +44,7 @@ bool replayCachedCompilerOutputs(
     llvm::cas::ObjectStore &CAS, llvm::cas::ActionCache &Cache,
     llvm::cas::ObjectRef BaseKey, DiagnosticEngine &Diag,
     const FrontendInputsAndOutputs &InputsAndOutputs,
-    CachingDiagnosticsProcessor &CDP, bool CacheRemarks);
+    CachingDiagnosticsProcessor &CDP, bool CacheRemarks, bool UseCASBackend);
 
 /// Load the cached compile result from cache.
 std::unique_ptr<llvm::MemoryBuffer> loadCachedCompileResultFromCacheKey(
diff --git a/lib/Frontend/CachingUtils.cpp b/lib/Frontend/CachingUtils.cpp
@@ -29,6 +29,7 @@
 #include "llvm/CAS/HierarchicalTreeBuilder.h"
 #include "llvm/CAS/ObjectStore.h"
 #include "llvm/CAS/TreeEntry.h"
+#include "llvm/MCCAS/MCCASObjectV1.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/OptTable.h"
 #include "llvm/Support/Debug.h"
@@ -134,7 +135,7 @@ lookupCacheKey(ObjectStore &CAS, ActionCache &Cache, ObjectRef CacheKey) {
 bool replayCachedCompilerOutputs(
     ObjectStore &CAS, ActionCache &Cache, ObjectRef BaseKey,
     DiagnosticEngine &Diag, const FrontendInputsAndOutputs &InputsAndOutputs,
-    CachingDiagnosticsProcessor &CDP, bool CacheRemarks) {
+    CachingDiagnosticsProcessor &CDP, bool CacheRemarks, bool UseCASBackend) {
   bool CanReplayAllOutput = true;
   struct OutputEntry {
     std::string Path;
@@ -143,6 +144,7 @@ bool replayCachedCompilerOutputs(
   };
   SmallVector<OutputEntry> OutputProxies;
   std::optional<OutputEntry> DiagnosticsOutput;
+  std::string ObjFile;
 
   auto replayOutputsForInputFile = [&](const std::string &InputPath,
                                        unsigned InputIndex,
@@ -176,7 +178,6 @@ bool replayCachedCompilerOutputs(
     CachedResultLoader Loader(CAS, **OutputRef);
     LLVM_DEBUG(llvm::dbgs() << "DEBUG: lookup cache key \'" << OutID.toString()
                             << "\' for input \'" << InputPath << "\n";);
-
     if (auto Err = Loader.replay([&](file_types::ID Kind,
                                      ObjectRef Ref) -> Error {
           auto OutputPath = Outputs.find(Kind);
@@ -188,6 +189,9 @@ bool replayCachedCompilerOutputs(
           if (!Proxy)
             return Proxy.takeError();
 
+          if (Kind == file_types::ID::TY_Object && UseCASBackend)
+            ObjFile = OutputPath->second;
+
           if (Kind == file_types::ID::TY_CachedDiagnostics) {
             assert(!DiagnosticsOutput && "more than 1 diagnotics found");
             DiagnosticsOutput = OutputEntry{OutputPath->second, OutID, *Proxy};
@@ -281,7 +285,14 @@ bool replayCachedCompilerOutputs(
                     toString(File.takeError()));
       continue;
     }
-    *File << Output.Proxy.getData();
+
+    if (UseCASBackend && Output.Path == ObjFile) {
+      auto Schema = std::make_unique<llvm::mccasformats::v1::MCSchema>(CAS);
+      if (auto E = Schema->serializeObjectFile(Output.Proxy, *File))
+        Diag.diagnose(SourceLoc(), diag::error_mccas, toString(std::move(E)));
+    } else
+      *File << Output.Proxy.getData();
+
     if (auto E = File->keep()) {
       Diag.diagnose(SourceLoc(), diag::error_closing_output, Output.Path,
                     toString(std::move(E)));
diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp
@@ -1478,7 +1478,8 @@ static bool tryReplayCompilerResults(CompilerInstance &Instance) {
       Instance.getObjectStore(), Instance.getActionCache(),
       *Instance.getCompilerBaseKey(), Instance.getDiags(),
       Instance.getInvocation().getFrontendOptions().InputsAndOutputs, *CDP,
-      Instance.getInvocation().getCASOptions().EnableCachingRemarks);
+      Instance.getInvocation().getCASOptions().EnableCachingRemarks,
+      Instance.getInvocation().getIRGenOptions().UseCASBackend);
 
   // If we didn't replay successfully, re-start capture.
   if (!replayed)
diff --git a/test/CAS/cache_replay_mccas.swift b/test/CAS/cache_replay_mccas.swift
@@ -0,0 +1,69 @@
+// REQUIRES: OS=macosx
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: %target-swift-frontend -scan-dependencies -module-name Test -O -module-cache-path %t/clang-module-cache \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \
+// RUN:   %t/test.swift -I %t -o %t/deps.json -cache-compile-job -cas-backend -cas-backend-mode=verify -cas-path %t/cas
+
+/// Check clang module
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:Dummy > %t/dummy.cmd
+// RUN: %swift_frontend_plain @%t/dummy.cmd -Rcache-compile-job 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
+// RUN: %swift_frontend_plain @%t/dummy.cmd -Rcache-compile-job 2>&1 | %FileCheck --check-prefix=CACHE-HIT-CLANG %s
+
+// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json
+// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid
+
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd
+// RUN: echo "\"-disable-implicit-string-processing-module-import\"" >> %t/MyApp.cmd
+// RUN: echo "\"-disable-implicit-concurrency-module-import\"" >> %t/MyApp.cmd
+// RUN: echo "\"-disable-implicit-swift-modules\"" >> %t/MyApp.cmd
+// RUN: echo "\"-parse-stdlib\"" >> %t/MyApp.cmd
+// RUN: echo "\"-explicit-swift-module-map-file\"" >> %t/MyApp.cmd
+// RUN: echo "\"@%t/map.casid\"" >> %t/MyApp.cmd
+
+/// Run the command first time, expect cache miss.
+// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c -emit-dependencies \
+// RUN:  -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
+// RUN: test -f %t/test.o
+// RUN: test -f %t/test.d
+// RUN: test -f %t/Test.swiftmodule
+
+/// Expect cache hit for second time.
+// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c -emit-dependencies \
+// RUN:  -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s
+// RUN: test -f %t/test.o
+// RUN: test -f %t/test.d
+// RUN: test -f %t/Test.swiftmodule
+
+/// Expect cache miss a subset of outputs.
+// RUN %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c \
+// RUN  -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
+
+/// Cache hit for retry.
+// RUN %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c \
+// RUN  -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s
+
+/// Skip cache
+// RUN %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job -cache-disable-replay %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c \
+// RUN  -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --allow-empty --check-prefix=SKIP-CACHE %s
+
+// CACHE-MISS: remark: cache miss for input
+// CACHE-HIT: remark: replay output file '<cached-diagnostics>': key 'llvmcas://{{.*}}'
+// CACHE-HIT: remark: replay output file '{{.*}}{{/|\\}}test.o': key 'llvmcas://{{.*}}'
+// CACHE-HIT: remark: replay output file '{{.*}}{{/|\\}}Test.swiftmodule': key 'llvmcas://{{.*}}'
+// CACHE-HIT-CLANG: remark: replay output file '<cached-diagnostics>': key 'llvmcas://{{.*}}'
+// CACHE-HIT-CLANG: remark: replay output file '{{.*}}{{/|\\}}Dummy-{{.*}}.pcm': key 'llvmcas://{{.*}}'
+// SKIP-CACHE-NOT: remark:
+
+//--- test.swift
+import Dummy
+func testFunc() {}
+
+//--- module.modulemap
+module Dummy {
+ umbrella header "Dummy.h"
+}
+
+//--- Dummy.h
+void dummy(void);
diff --git a/test/CAS/cache_replay_multiple_files_mccas.swift b/test/CAS/cache_replay_multiple_files_mccas.swift
@@ -0,0 +1,48 @@
+// REQUIRES: OS=macosx
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: %target-swift-frontend -scan-dependencies -module-name Test -O \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \
+// RUN:   %t/test.swift %t/foo.swift -o %t/deps.json -cache-compile-job -cas-backend -cas-backend-mode=verify -cas-path %t/cas
+
+// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json
+// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid
+
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd
+// RUN: echo "\"-disable-implicit-string-processing-module-import\"" >> %t/MyApp.cmd
+// RUN: echo "\"-disable-implicit-concurrency-module-import\"" >> %t/MyApp.cmd
+// RUN: echo "\"-parse-stdlib\"" >> %t/MyApp.cmd
+
+/// Test compile multiple inputs with batch mode.
+// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift %t/foo.swift -emit-module -o %t/Test.swiftmodule \
+// RUN:  -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
+// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job -primary-file %t/test.swift %t/foo.swift -c -o %t/test.o  \
+// RUN:  -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
+// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -primary-file %t/foo.swift -c -o %t/foo.o  \
+// RUN:  -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
+// RUN: test -f %t/test.o
+// RUN: test -f %t/foo.o
+// RUN: test -f %t/Test.swiftmodule
+
+/// Expect cache hit second time
+// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift %t/foo.swift -emit-module -o %t/Test.swiftmodule \
+// RUN:  -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s
+// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job -primary-file %t/test.swift %t/foo.swift -c -o %t/test.o  \
+// RUN:  -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s
+// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -primary-file %t/foo.swift -c -o %t/foo.o  \
+// RUN:  -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s
+// RUN: test -f %t/test.o
+// RUN: test -f %t/foo.o
+// RUN: test -f %t/Test.swiftmodule
+
+//--- test.swift
+func testFunc() {}
+
+//--- foo.swift
+func foo() {}
+
+// CACHE-MISS: remark: cache miss for input
+// CACHE-MISS-NOT: remark: replay output file
+// CACHE-HIT: remark: replay output file
+// CACHE-HIT-NOT: remark: cache miss for input
