[Caching] Teach libSwiftScan to replay all diagnostics kind

Add support for serialized diagnostics, parseable output, and other
kinds of output from diagnostics engine to the libSwiftScan
replayCompilation API.

rdar://129015959

Author: cachemeifyoucan

include/swift/Frontend/DiagnosticHelper.h

@@ -18,6 +18,7 @@
 #define SWIFT_FRONTEND_DIAGNOSTIC_HELPER_H
 
 #include "swift/Basic/LLVM.h"
+#include "llvm/Support/raw_ostream.h"
 
 namespace swift {
 class CompilerInstance;
@@ -30,7 +31,10 @@ class DiagnosticHelper {
 
 public:
   /// Create a DiagnosticHelper class to emit diagnostics from frontend actions.
+  /// OS is the stream to print diagnostics. useQuasiPID determines if using
+  /// real PID when priting parseable output.
   static DiagnosticHelper create(CompilerInstance &instance,
+                                 llvm::raw_pwrite_stream &OS = llvm::errs(),
                                  bool useQuasiPID = false);
 
   /// Initialized all DiagConsumers and add to the CompilerInstance.
@@ -57,7 +61,8 @@ class DiagnosticHelper {
   ~DiagnosticHelper();
 
 private:
-  DiagnosticHelper(CompilerInstance &instance, bool useQuasiPID);
+  DiagnosticHelper(CompilerInstance &instance, llvm::raw_pwrite_stream &OS,
+                   bool useQuasiPID);
 };
 
 } // namespace swift

lib/Frontend/DiagnosticHelper.cpp

@@ -16,6 +16,7 @@
 
 #include "swift/Frontend/DiagnosticHelper.h"
 #include "swift/AST/DiagnosticEngine.h"
+#include "swift/AST/DiagnosticsFrontend.h"
 #include "swift/Basic/Edit.h"
 #include "swift/Basic/ParseableOutput.h"
 #include "swift/Basic/SourceManager.h"
@@ -24,6 +25,7 @@
 #include "swift/Frontend/PrintingDiagnosticConsumer.h"
 #include "swift/Frontend/SerializedDiagnosticConsumer.h"
 #include "swift/Migrator/FixitFilter.h"
+#include "llvm/Support/raw_ostream.h"
 
 #if __has_include(<unistd.h>)
 #include <unistd.h>
@@ -38,7 +40,8 @@ class LLVM_LIBRARY_VISIBILITY DiagnosticHelper::Implementation {
   friend class DiagnosticHelper;
 
 public:
-  Implementation(CompilerInstance &instance, bool useQuasiPID);
+  Implementation(CompilerInstance &instance, llvm::raw_pwrite_stream &OS,
+                 bool useQuasiPID);
 
   void initDiagConsumers(CompilerInvocation &invocation);
 
@@ -59,6 +62,7 @@ class LLVM_LIBRARY_VISIBILITY DiagnosticHelper::Implementation {
   const sys::TaskProcessInformation procInfo;
 
   CompilerInstance &instance;
+  llvm::raw_pwrite_stream &errOS;
 
   // potentially created diagnostic consumers.
   PrintingDiagnosticConsumer PDC;
@@ -226,9 +230,10 @@ createJSONFixItDiagnosticConsumerIfNeeded(
 }
 
 DiagnosticHelper::Implementation::Implementation(CompilerInstance &instance,
+                                                 llvm::raw_pwrite_stream &OS,
                                                  bool useQuasiPID)
     : OSPid(useQuasiPID ? QUASI_PID_START : getpid()), procInfo(OSPid),
-      instance(instance) {
+      instance(instance), errOS(OS), PDC(OS) {
   instance.addDiagnosticConsumer(&PDC);
 }
 
@@ -396,7 +401,7 @@ void DiagnosticHelper::Implementation::beginMessage(
         [&](const InputFile &Input, unsigned idx) -> bool {
           ArrayRef<InputFile> Inputs(Input);
           emitBeganMessage(
-              llvm::errs(), mapFrontendInvocationToAction(invocation),
+              errOS, mapFrontendInvocationToAction(invocation),
               constructDetailedTaskDescription(invocation, Inputs, args),
               pid - idx, procInfo);
           return false;
@@ -405,7 +410,7 @@ void DiagnosticHelper::Implementation::beginMessage(
     // If no primary inputs are present, we are in WMO or EmitModule.
     bool isEmitModule = invocation.getFrontendOptions().RequestedAction ==
                         FrontendOptions::ActionType::EmitModuleOnly;
-    emitBeganMessage(llvm::errs(), mapFrontendInvocationToAction(invocation),
+    emitBeganMessage(errOS, mapFrontendInvocationToAction(invocation),
                      constructDetailedTaskDescription(
                          invocation, IO.getAllInputs(), args, isEmitModule),
                      OSPid, procInfo);
@@ -438,7 +443,7 @@ void DiagnosticHelper::Implementation::endMessage(int retCode) {
       std::copy(PrimaryDiags.begin(), PrimaryDiags.end(),
                 std::ostream_iterator<std::string>(JoinedDiags, Delim));
 
-      emitFinishedMessage(llvm::errs(),
+      emitFinishedMessage(errOS,
                           mapFrontendInvocationToAction(invocation),
                           JoinedDiags.str(), retCode, pid - idx, procInfo);
       return false;
@@ -455,7 +460,7 @@ void DiagnosticHelper::Implementation::endMessage(int retCode) {
     std::ostringstream JoinedDiags;
     std::copy(AllDiagnostics.begin(), AllDiagnostics.end(),
               std::ostream_iterator<std::string>(JoinedDiags, Delim));
-    emitFinishedMessage(llvm::errs(), mapFrontendInvocationToAction(invocation),
+    emitFinishedMessage(errOS, mapFrontendInvocationToAction(invocation),
                         JoinedDiags.str(), retCode, OSPid, procInfo);
   }
 
@@ -492,12 +497,15 @@ void DiagnosticHelper::Implementation::diagnoseFatalError(const char *reason,
 }
 
 DiagnosticHelper DiagnosticHelper::create(CompilerInstance &instance,
+                                          llvm::raw_pwrite_stream &OS,
                                           bool useQuasiPID) {
-  return DiagnosticHelper(instance, useQuasiPID);
+  return DiagnosticHelper(instance, OS, useQuasiPID);
 }
 
-DiagnosticHelper::DiagnosticHelper(CompilerInstance &instance, bool useQuasiPID)
-    : Impl(*new Implementation(instance, useQuasiPID)) {}
+DiagnosticHelper::DiagnosticHelper(CompilerInstance &instance,
+                                   llvm::raw_pwrite_stream &OS,
+                                   bool useQuasiPID)
+    : Impl(*new Implementation(instance, OS, useQuasiPID)) {}
 
 DiagnosticHelper::~DiagnosticHelper() { delete &Impl; }
 

test/CAS/swift-scan-diagnostics-batch.swift

@@ -0,0 +1,63 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: %target-swift-frontend -scan-dependencies -module-name Test -O \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \
+// RUN:   %t/a.swift %t/b.swift %t/c.swift -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas
+
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd
+// RUN: echo "\"-disable-implicit-string-processing-module-import\"" >> %t/MyApp.cmd
+// RUN: echo "\"-disable-implicit-concurrency-module-import\"" >> %t/MyApp.cmd
+// RUN: echo "\"-parse-stdlib\"" >> %t/MyApp.cmd
+
+// RUN: %swift-scan-test -action compute_cache_key_from_index -cas-path %t/cas -input 0 -- \
+// RUN:   %target-swift-frontend -cache-compile-job -primary-file %t/a.swift -primary-file %t/b.swift %t/c.swift \
+// RUN:   -c -emit-dependencies -module-name Test -o %t/a.o -o %t/b.o -cas-path %t/cas \
+// RUN:   @%t/MyApp.cmd > %t/key0.casid
+
+// RUN: %swift-scan-test -action compute_cache_key_from_index -cas-path %t/cas -input 1 -- \
+// RUN:   %target-swift-frontend -cache-compile-job -primary-file %t/a.swift -primary-file %t/b.swift %t/c.swift \
+// RUN:   -c -emit-dependencies -module-name Test -o %t/a.o -o %t/b.o -cas-path %t/cas \
+// RUN:   @%t/MyApp.cmd > %t/key1.casid
+
+// RUN: %target-swift-frontend -cache-compile-job \
+// RUN:  -primary-file %t/a.swift -primary-file %t/b.swift %t/c.swift \
+// RUN:  -c -emit-dependencies \
+// RUN:  -module-name Test -o %t/a.o -o %t/b.o -cas-path %t/cas @%t/MyApp.cmd
+
+// RUN: %target-swift-frontend -cache-compile-job \
+// RUN:  %t/a.swift %t/b.swift -primary-file %t/c.swift \
+// RUN:  -c -emit-dependencies \
+// RUN:  -module-name Test -o %t/c.o -cas-path %t/cas @%t/MyApp.cmd
+
+// RUN: %swift-scan-test -action replay_result -cas-path %t/cas -id @%t/key0.casid -- %target-swift-frontend -cache-compile-job \
+// RUN:   -primary-file %t/a.swift -primary-file %t/b.swift  %t/c.swift \
+// RUN:   -c -emit-dependencies -module-name Test -o %t/a2.o -o %t/b2.o -cas-path %t/cas \
+// RUN:   -serialize-diagnostics -serialize-diagnostics-path %t/a2.dia -serialize-diagnostics-path %t/b2.dia \
+// RUN:   @%t/MyApp.cmd -frontend-parseable-output 2>&1 | %FileCheck %s --check-prefix=PARSEABLE
+
+// RUN: c-index-test -read-diagnostics %t/a2.dia 2>&1 | %FileCheck %s --check-prefix=A-WARN
+// RUN: c-index-test -read-diagnostics %t/b2.dia 2>&1 | %FileCheck %s --check-prefix=B-WARN
+
+// A-WARN: warning: This is a warning
+// B-WARN: warning: This is also a warning
+
+// RUN: %swift-scan-test -action replay_result -cas-path %t/cas -id @%t/key1.casid -- %target-swift-frontend -cache-compile-job \
+// RUN:   -primary-file %t/a.swift -primary-file %t/b.swift  %t/c.swift \
+// RUN:   -c -emit-dependencies -module-name Test -o %t/a2.o -o %t/b2.o -cas-path %t/cas \
+// RUN:   -serialize-diagnostics -serialize-diagnostics-path %t/a2.dia -serialize-diagnostics-path %t/b2.dia \
+// RUN:   @%t/MyApp.cmd -frontend-parseable-output 2>&1 | %FileCheck %s --check-prefix=NO-OUTPUT --allow-empty
+
+// PARSEABLE-COUNT-1:   "kind": "began",
+// PARSEABLE-COUNT-1:   "kind": "finished",
+// PRASEABLE-NOT: "kind": "began",
+// PRASEABLE-NOT: "kind": "finished",
+
+// NO-OUTPUT-NOT: "kind": "began",
+
+//--- a.swift
+#warning("This is a warning")
+//--- b.swift
+#warning("This is also a warning")
+//--- c.swift
+#warning("This is another warning")

test/CAS/swift-scan-diagnostics.swift

@@ -0,0 +1,62 @@
+// RUN: %empty-directory(%t)
+
+// RUN: %target-swift-frontend -scan-dependencies -module-name Test -O \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \
+// RUN:   %s -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas
+
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd
+// RUN: echo "\"-disable-implicit-string-processing-module-import\"" >> %t/MyApp.cmd
+// RUN: echo "\"-disable-implicit-concurrency-module-import\"" >> %t/MyApp.cmd
+// RUN: echo "\"-parse-stdlib\"" >> %t/MyApp.cmd
+
+// RUN: %swift-scan-test -action compute_cache_key -cas-path %t/cas -input %s -- %target-swift-frontend -cache-compile-job %s \
+// RUN:   -emit-module -emit-module-path %t/Test.swiftmodule -c -emit-dependencies -module-name Test -o %t/test.o -cas-path %t/cas \
+// RUN:   @%t/MyApp.cmd > %t/key.casid
+
+// RUN: %swift-scan-test -action compute_cache_key_from_index -cas-path %t/cas -input 0 -- %target-swift-frontend -cache-compile-job %s \
+// RUN:   -emit-module -emit-module-path %t/Test.swiftmodule -c -emit-dependencies -module-name Test -o %t/test.o -cas-path %t/cas \
+// RUN:   @%t/MyApp.cmd > %t/key1.casid
+
+// RUN: diff %t/key.casid %t/key1.casid
+
+// RUN: not %swift-scan-test -action cache_query -id @%t/key.casid -cas-path %t/cas 2>&1 | %FileCheck %s --check-prefix=CHECK-QUERY-NOT-FOUND
+
+// RUN: %target-swift-frontend -cache-compile-job %s -emit-module -emit-module-path %t/Test.swiftmodule -c -emit-dependencies \
+// RUN:  -serialize-diagnostics -serialize-diagnostics-path %t/test.dia \
+// RUN:  -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd
+
+// RUN: %swift-scan-test -action cache_query -id @%t/key.casid -cas-path %t/cas | %FileCheck %s --check-prefix=CHECK-QUERY
+
+// RUN: %swift-scan-test -action replay_result -cas-path %t/cas -id @%t/key.casid -- %target-swift-frontend -cache-compile-job %s \
+// RUN:   -emit-module -emit-module-path %t/Test2.swiftmodule -c -emit-dependencies -module-name Test -o %t/test2.o -cas-path %t/cas \
+// RUN:   -serialize-diagnostics -serialize-diagnostics-path %t/test2.dia \
+// RUN:   @%t/MyApp.cmd
+
+// RUN: diff %t/Test.swiftmodule %t/Test2.swiftmodule
+// RUN: diff %t/test.o %t/test2.o
+// RUN: diff %t/test.dia %t/test2.dia
+
+// CHECK-QUERY-NOT-FOUND: cached output not found
+// CHECK-QUERY: Cached Compilation for key "llvmcas://{{.*}}" has 4 outputs:
+// CHECK-QUERY-NEXT: object: llvmcas://
+// CHECK-QUERY-NEXT: dependencies: llvmcas://
+// CHECK-QUERY-NEXT: swiftmodule: llvmcas://
+// CHECK-QUERY-NEXT: cached-diagnostics: llvmcas://
+
+// RUN: %swift-scan-test -action replay_result -cas-path %t/cas -id @%t/key.casid -- %target-swift-frontend -cache-compile-job %s \
+// RUN:   -emit-module -emit-module-path %t/Test2.swiftmodule -c -emit-dependencies -module-name Test -o %t/test2.o -cas-path %t/cas \
+// RUN:   -frontend-parseable-output -serialize-diagnostics -serialize-diagnostics-path %t/test3.dia \
+// RUN:   @%t/MyApp.cmd 2>&1 | %FileCheck %s --check-prefix=PARSEABLE
+
+// RUN: diff %t/test.dia %t/test3.dia
+
+// PARSEABLE: {{[1-9][0-9]*}}
+// PARSEABLE-NEXT: {
+// PARSEABLE-NEXT:   "kind": "began",
+// PARSEABLE-NEXT:   "name": "compile",
+
+// PARSEABLE:   "kind": "finished",
+// PARSEABLE-NEXT:   "name": "compile",
+
+
+#warning("This is a warning")

test/CAS/swift-scan-test.swift

@@ -31,7 +31,7 @@
 // RUN:   @%t/MyApp.cmd
 
 // RUN: diff %t/Test.swiftmodule %t/Test2.swiftmodule
-// RUN: diff %t/test.o %t/test.o
+// RUN: diff %t/test.o %t/test2.o
 
 // CHECK-QUERY-NOT-FOUND: cached output not found
 // CHECK-QUERY: Cached Compilation for key "llvmcas://{{.*}}" has 4 outputs:

tools/libSwiftScan/SwiftCaching.cpp

@@ -26,6 +26,7 @@
 #include "swift/Frontend/CachingUtils.h"
 #include "swift/Frontend/CompileJobCacheKey.h"
 #include "swift/Frontend/CompileJobCacheResult.h"
+#include "swift/Frontend/DiagnosticHelper.h"
 #include "swift/Frontend/Frontend.h"
 #include "swift/Frontend/PrintingDiagnosticConsumer.h"
 #include "swift/Option/Options.h"
@@ -917,19 +918,7 @@ static llvm::Error replayCompilation(SwiftScanReplayInstance &Instance,
   const auto &Input = AllInputs[Comp.InputIndex];
 
   // Setup DiagnosticsConsumers.
-  // FIXME: Reduce code duplication against `performFrontend()` and add support
-  // for JSONFIXIT, SerializedDiagnostics, etc.
-  PrintingDiagnosticConsumer PDC(Err);
-  Inst.addDiagnosticConsumer(&PDC);
-
-  if (Invocation.getDiagnosticOptions().UseColor)
-    PDC.forceColors();
-  PDC.setPrintEducationalNotes(
-      Invocation.getDiagnosticOptions().PrintEducationalNotes);
-  PDC.setFormattingStyle(
-      Invocation.getDiagnosticOptions().PrintedFormattingStyle);
-  PDC.setEmitMacroExpansionFiles(
-      Invocation.getDiagnosticOptions().EmitMacroExpansionFiles);
+  DiagnosticHelper DH = DiagnosticHelper::create(Inst, Err, /*QuasiPID=*/true);
 
   std::string InstanceSetupError;
   if (Inst.setupForReplay(Instance.Invocation, InstanceSetupError,
@@ -989,15 +978,29 @@ static llvm::Error replayCompilation(SwiftScanReplayInstance &Instance,
   // Replay diagnostics first.
   // FIXME: Currently, the diagnostics is replay from the first file.
   if (DiagnosticsOutput) {
+    DH.initDiagConsumers(Invocation);
+    DH.beginMessage(Invocation, Instance.Args);
+
     if (auto E = CDP->replayCachedDiagnostics(DiagnosticsOutput->getData()))
       return E;
 
     if (Remarks)
       Inst.getDiags().diagnose(SourceLoc(), diag::replay_output,
                                "<cached-diagnostics>",
                                CAS.getID(Comp.Key).toString());
+  } else {
+    // Don't write anything when parseable output is requested.
+    if (Invocation.getFrontendOptions().FrontendParseableOutput)
+      DH.setSuppressOutput(true);
   }
 
+  SWIFT_DEFER {
+    if (DiagnosticsOutput) {
+      DH.endMessage(0);
+      Inst.getDiags().finishProcessing();
+    }
+  };
+
   // OutputBackend for replay.
   ReplayOutputBackend Backend(
       makeIntrusiveRefCnt<llvm::vfs::OnDiskOutputBackend>(), Out);

tools/swift-scan-test/swift-scan-test.cpp

@@ -151,8 +151,13 @@ static int action_replay_result(swiftscan_cas_t cas, const char *key,
                                 std::vector<const char *> &Args) {
   swiftscan_string_ref_t err_msg;
   auto comp = swiftscan_cache_query(cas, key, /*globally=*/false, &err_msg);
-  if (!comp)
-    return printError(err_msg);
+  if (!comp) {
+    if (err_msg.length != 0)
+      return printError(err_msg);
+
+    llvm::errs() << "key " << key << " not found for replay\n";
+    return EXIT_FAILURE;
+  }
 
   SWIFT_DEFER { swiftscan_cached_compilation_dispose(comp); };
   auto numOutput = swiftscan_cached_compilation_get_num_outputs(comp);