[Dependency Scanning] Configure a thread-safe serialized diagnostics consumer for in-memory scans

This change introduces a thread-safe version of the 'SerializedDiagnosticConsumer' and refactors scanning compilation instance creation code to ensure this consumer gets added when the scanner query configuration command-line includes '-serialized-diagnostics-path' option.

Author: artemcm

include/swift/DependencyScan/DependencyScanningTool.h

@@ -15,20 +15,34 @@
 
 #include "swift-c/DependencyScan/DependencyScan.h"
 #include "swift/Frontend/Frontend.h"
+#include "swift/AST/DiagnosticConsumer.h"
 #include "swift/AST/ModuleDependencies.h"
 #include "swift/DependencyScan/ScanDependencies.h"
 #include "swift/Frontend/PrintingDiagnosticConsumer.h"
+#include "swift/Frontend/SerializedDiagnosticConsumer.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/StringSaver.h"
 
 namespace swift {
 namespace dependencies {
 class DependencyScanningTool;
-class DependencyScanDiagnosticCollector;
-
-struct ScanQueryInstance {
-  std::unique_ptr<CompilerInstance> ScanInstance;
-  std::shared_ptr<DependencyScanDiagnosticCollector> ScanDiagnostics;
+class DepScanInMemoryDiagnosticCollector;
+
+struct ScanQueryContext {
+  /// Primary CompilerInstance configured for this scanning action
+  llvm::ErrorOr<std::unique_ptr<CompilerInstance>> ScanInstance;
+  /// An thread-safe diagnostic consumer which collects all emitted
+  /// diagnostics in the scan to be reporte via libSwiftScan API
+  std::unique_ptr<DepScanInMemoryDiagnosticCollector> InMemoryDiagnosticCollector;
+  /// A thread-safe serialized diagnostics consumer.
+  /// Note, although type-erased, this must be an instance of
+  /// 'ThreadSafeSerializedDiagnosticConsumer'
+  std::unique_ptr<DiagnosticConsumer> SerializedDiagnosticConsumer;
+
+  ~ScanQueryContext() {
+    if (SerializedDiagnosticConsumer)
+      SerializedDiagnosticConsumer->finishProcessing();
+  }
 };
 
 /// Pure virtual Diagnostic consumer intended for collecting
@@ -47,12 +61,16 @@ class ThreadSafeDiagnosticCollector : public DiagnosticConsumer {
 };
 
 /// Diagnostic consumer that simply collects the diagnostics emitted so-far
-class DependencyScanDiagnosticCollector : public ThreadSafeDiagnosticCollector {
+/// and uses a representation agnostic from any specific CompilerInstance state
+/// which may have been used to emit the diagnostic
+class DepScanInMemoryDiagnosticCollector
+    : public ThreadSafeDiagnosticCollector {
 private:
   struct ScannerDiagnosticInfo {
     std::string Message;
     llvm::SourceMgr::DiagKind Severity;
-    std::optional<ScannerImportStatementInfo::ImportDiagnosticLocationInfo> ImportLocation;
+    std::optional<ScannerImportStatementInfo::ImportDiagnosticLocationInfo>
+        ImportLocation;
   };
   std::vector<ScannerDiagnosticInfo> Diagnostics;
 
@@ -61,7 +79,7 @@ class DependencyScanDiagnosticCollector : public ThreadSafeDiagnosticCollector {
 
 public:
   friend DependencyScanningTool;
-  DependencyScanDiagnosticCollector() {}
+  DepScanInMemoryDiagnosticCollector() {}
   void reset() { Diagnostics.clear(); }
   const std::vector<ScannerDiagnosticInfo> &getDiagnostics() const {
     return Diagnostics;
@@ -97,10 +115,9 @@ class DependencyScanningTool {
 
   /// Using the specified invocation command, instantiate a CompilerInstance
   /// that will be used for this scan.
-  llvm::ErrorOr<ScanQueryInstance>
-  initCompilerInstanceForScan(ArrayRef<const char *> Command,
-                              StringRef WorkingDirectory,
-                              std::shared_ptr<DependencyScanDiagnosticCollector> scannerDiagnosticsCollector);
+  ScanQueryContext
+  createScanQueryContext(ArrayRef<const char *> Command,
+                         StringRef WorkingDirectory);
 
 private:
   /// Shared cache of module dependencies, re-used by individual full-scan queries
@@ -113,7 +130,7 @@ class DependencyScanningTool {
   llvm::StringSaver Saver;
 };
 
-swiftscan_diagnostic_set_t *mapCollectedDiagnosticsForOutput(const DependencyScanDiagnosticCollector *diagnosticCollector);
+swiftscan_diagnostic_set_t *mapCollectedDiagnosticsForOutput(const DepScanInMemoryDiagnosticCollector *diagnosticCollector);
 
 } // end namespace dependencies
 } // end namespace swift

include/swift/DependencyScan/ScanDependencies.h

@@ -43,7 +43,7 @@ using ModuleDependencyIDSet =
 class SwiftDependencyScanningService;
 
 namespace dependencies {
-class DependencyScanDiagnosticCollector;
+struct ScanQueryContext;
 
 using CompilerArgInstanceCacheMap =
     llvm::StringMap<std::tuple<std::unique_ptr<CompilerInstance>,
@@ -62,16 +62,15 @@ bool prescanDependencies(CompilerInstance &instance);
 /// Scans the dependencies of the main module of \c instance.
 llvm::ErrorOr<swiftscan_dependency_graph_t>
 performModuleScan(SwiftDependencyScanningService &service,
-                  CompilerInstance &instance,
                   ModuleDependenciesCache &cache,
-                  DependencyScanDiagnosticCollector *diagnostics = nullptr);
+                  ScanQueryContext &queryContext);
 
 /// Scans the main module of \c instance for all direct module imports
 llvm::ErrorOr<swiftscan_import_set_t>
 performModulePrescan(SwiftDependencyScanningService &service,
-                     CompilerInstance &instance,
                      ModuleDependenciesCache &cache,
-                     DependencyScanDiagnosticCollector *diagnostics = nullptr);
+                     ScanQueryContext &queryContext);
+
 
 namespace incremental {
 /// For the given module dependency graph captured in the 'cache',

include/swift/Frontend/SerializedDiagnosticConsumer.h

@@ -37,6 +37,15 @@ namespace swift {
     /// \returns A new diagnostic consumer that serializes diagnostics.
     std::unique_ptr<DiagnosticConsumer>
     createConsumer(llvm::StringRef outputPath, bool emitMacroExpansionFiles);
+
+    /// Create a thread-safe DiagnosticConsumer that serializes diagnostics to a file,
+    /// using Clang's serialized diagnostics format.
+    ///
+    /// \param outputPath the file path to write the diagnostics to.
+    ///
+    /// \returns A new diagnostic consumer that serializes diagnostics.
+    std::unique_ptr<DiagnosticConsumer>
+    createThreadSafeConsumer(llvm::StringRef outputPath, bool emitMacroExpansionFiles);
   }
 }
 

lib/DependencyScan/DependencyScanningTool.cpp

@@ -73,7 +73,7 @@ void ThreadSafeDiagnosticCollector::handleDiagnostic(SourceManager &SM,
   }
 }
 
-void DependencyScanDiagnosticCollector::addDiagnostic(
+void DepScanInMemoryDiagnosticCollector::addDiagnostic(
     SourceManager &SM, const DiagnosticInfo &Info) {
   // Determine what kind of diagnostic we're emitting.
   llvm::SourceMgr::DiagKind SMKind;
@@ -131,7 +131,7 @@ void DependencyScanDiagnosticCollector::addDiagnostic(
 }
 
 swiftscan_diagnostic_set_t *mapCollectedDiagnosticsForOutput(
-    const DependencyScanDiagnosticCollector *diagnosticCollector) {
+    const DepScanInMemoryDiagnosticCollector *diagnosticCollector) {
   auto collectedDiagnostics = diagnosticCollector->getDiagnostics();
   auto numDiagnostics = collectedDiagnostics.size();
   swiftscan_diagnostic_set_t *diagnosticOutput = new swiftscan_diagnostic_set_t;
@@ -183,7 +183,7 @@ swiftscan_diagnostic_set_t *mapCollectedDiagnosticsForOutput(
 // module dependnecies but captures the diagnostics emitted during the attempted
 // scan query.
 static swiftscan_dependency_graph_t generateHollowDiagnosticOutput(
-    const DependencyScanDiagnosticCollector &ScanDiagnosticConsumer) {
+    const DepScanInMemoryDiagnosticCollector &ScanDiagnosticConsumer) {
   // Create a dependency graph instance
   swiftscan_dependency_graph_t hollowResult = new swiftscan_dependency_graph_s;
 
@@ -254,7 +254,7 @@ static swiftscan_dependency_graph_t generateHollowDiagnosticOutput(
 // imports but captures the diagnostics emitted during the attempted
 // scan query.
 static swiftscan_import_set_t generateHollowDiagnosticOutputImportSet(
-    const DependencyScanDiagnosticCollector &ScanDiagnosticConsumer) {
+    const DepScanInMemoryDiagnosticCollector &ScanDiagnosticConsumer) {
   // Create an dependency graph instance
   swiftscan_import_set_t hollowResult = new swiftscan_import_set_s;
   hollowResult->imports = c_string_utils::create_empty_set();
@@ -268,72 +268,57 @@ DependencyScanningTool::DependencyScanningTool()
       Alloc(), Saver(Alloc) {}
 
 llvm::ErrorOr<swiftscan_dependency_graph_t>
-DependencyScanningTool::getDependencies(
-    ArrayRef<const char *> Command,
-    StringRef WorkingDirectory) {
-  // There may be errors as early as in instance initialization, so we must ensure
-  // we can catch those.
-  auto ScanDiagnosticConsumer =
-    std::make_shared<DependencyScanDiagnosticCollector>();
-
+DependencyScanningTool::getDependencies(ArrayRef<const char *> Command,
+                                        StringRef WorkingDirectory) {
   // The primary instance used to scan the query Swift source-code
-  auto QueryContextOrErr = initCompilerInstanceForScan(Command,
-                                                       WorkingDirectory,
-                                                       ScanDiagnosticConsumer);
-  if (QueryContextOrErr.getError())
-    return generateHollowDiagnosticOutput(*ScanDiagnosticConsumer);
-
-  auto QueryContext = std::move(*QueryContextOrErr);
+  auto QueryContext = createScanQueryContext(Command, WorkingDirectory);
+  if (QueryContext.ScanInstance.getError())
+    return generateHollowDiagnosticOutput(
+        *QueryContext.InMemoryDiagnosticCollector);
+  auto ScanInstance = QueryContext.ScanInstance->get();
 
   // Local scan cache instance, wrapping the shared global cache.
   ModuleDependenciesCache cache(
-      QueryContext.ScanInstance->getMainModule()->getNameStr().str(),
-      QueryContext.ScanInstance->getInvocation().getModuleScanningHash());
+      ScanInstance->getMainModule()->getNameStr().str(),
+      ScanInstance->getInvocation().getModuleScanningHash());
   // Execute the scanning action, retrieving the in-memory result
-  auto DependenciesOrErr = performModuleScan(*ScanningService,
-                                             *QueryContext.ScanInstance.get(),
-                                             cache,
-                                             QueryContext.ScanDiagnostics.get());
+  auto DependenciesOrErr =
+      performModuleScan(*ScanningService, cache, QueryContext);
+
   if (DependenciesOrErr.getError())
-    return generateHollowDiagnosticOutput(*ScanDiagnosticConsumer);
+    return generateHollowDiagnosticOutput(
+        *QueryContext.InMemoryDiagnosticCollector);
 
   return std::move(*DependenciesOrErr);
 }
 
 llvm::ErrorOr<swiftscan_import_set_t>
 DependencyScanningTool::getImports(ArrayRef<const char *> Command,
                                    StringRef WorkingDirectory) {
-  // There may be errors as early as in instance initialization, so we must ensure
-  // we can catch those
-  auto ScanDiagnosticConsumer = std::make_shared<DependencyScanDiagnosticCollector>();
   // The primary instance used to scan the query Swift source-code
-  auto QueryContextOrErr = initCompilerInstanceForScan(Command,
-                                                       WorkingDirectory,
-                                                       ScanDiagnosticConsumer);
-  if (QueryContextOrErr.getError())
-    return generateHollowDiagnosticOutputImportSet(*ScanDiagnosticConsumer);
-
-  auto QueryContext = std::move(*QueryContextOrErr);
+  auto QueryContext = createScanQueryContext(Command, WorkingDirectory);
+  if (QueryContext.ScanInstance.getError())
+    return generateHollowDiagnosticOutputImportSet(
+        *QueryContext.InMemoryDiagnosticCollector);
+  auto ScanInstance = QueryContext.ScanInstance->get();
 
   // Local scan cache instance, wrapping the shared global cache.
   ModuleDependenciesCache cache(
-      QueryContext.ScanInstance->getMainModule()->getNameStr().str(),
-      QueryContext.ScanInstance->getInvocation().getModuleScanningHash());
-  auto DependenciesOrErr = performModulePrescan(*ScanningService,
-                                                *QueryContext.ScanInstance.get(),
-                                                cache,
-                                                QueryContext.ScanDiagnostics.get());
+      ScanInstance->getMainModule()->getNameStr().str(),
+      ScanInstance->getInvocation().getModuleScanningHash());
+  // Execute the pre-scanning action, retrieving the in-memory result
+  auto DependenciesOrErr =
+      performModulePrescan(*ScanningService, cache, QueryContext);
+
   if (DependenciesOrErr.getError())
-    return generateHollowDiagnosticOutputImportSet(*ScanDiagnosticConsumer);
+    return generateHollowDiagnosticOutputImportSet(
+        *QueryContext.InMemoryDiagnosticCollector);
 
   return std::move(*DependenciesOrErr);
 }
 
-llvm::ErrorOr<ScanQueryInstance>
-DependencyScanningTool::initCompilerInstanceForScan(
-    ArrayRef<const char *> CommandArgs,
-    StringRef WorkingDir,
-    std::shared_ptr<DependencyScanDiagnosticCollector> scannerDiagnosticsCollector) {
+ScanQueryContext DependencyScanningTool::createScanQueryContext(
+    ArrayRef<const char *> CommandArgs, StringRef WorkingDir) {
   // The remainder of this method operates on shared state in the
   // scanning service
   llvm::sys::SmartScopedLock<true> Lock(DependencyScanningToolStateLock);
@@ -342,14 +327,20 @@ DependencyScanningTool::initCompilerInstanceForScan(
   // client-side API plumbing.
   llvm::sys::SmartScopedLock<true> TargetInfoLock(TargetInfoMutex);
 
+  // There may be errors as early as in instance initialization, so we must
+  // ensure we can catch those
+  auto ScannerDiagnosticsCollector =
+      std::make_unique<DepScanInMemoryDiagnosticCollector>();
+
   // State unique to an individual scan
   auto Instance = std::make_unique<CompilerInstance>();
-  Instance->addDiagnosticConsumer(scannerDiagnosticsCollector.get());
+  Instance->addDiagnosticConsumer(ScannerDiagnosticsCollector.get());
 
   // Basic error checking on the arguments
   if (CommandArgs.empty()) {
     Instance->getDiags().diagnose(SourceLoc(), diag::error_no_frontend_args);
-    return std::make_error_code(std::errc::invalid_argument);
+    return ScanQueryContext{std::make_error_code(std::errc::invalid_argument),
+                            std::move(ScannerDiagnosticsCollector), nullptr};
   }
 
   CompilerInvocation Invocation;
@@ -375,24 +366,39 @@ DependencyScanningTool::initCompilerInstanceForScan(
 
   if (Invocation.parseArgs(Args, Instance->getDiags(),
                            nullptr, WorkingDirectory, "/tmp/foo")) {
-    return std::make_error_code(std::errc::invalid_argument);
+    return ScanQueryContext{std::make_error_code(std::errc::invalid_argument),
+                            std::move(ScannerDiagnosticsCollector), nullptr};
   }
 
   // Setup the instance
   std::string InstanceSetupError;
-  if (Instance->setup(Invocation, InstanceSetupError)) {
-    return std::make_error_code(std::errc::not_supported);
-  }
+  if (Instance->setup(Invocation, InstanceSetupError))
+    return ScanQueryContext{std::make_error_code(std::errc::not_supported),
+                            std::move(ScannerDiagnosticsCollector), nullptr};
+
   Invocation.getFrontendOptions().LLVMArgs.clear();
 
   // Setup the caching service after the instance finishes setup.
   if (ScanningService->setupCachingDependencyScanningService(*Instance))
-    return std::make_error_code(std::errc::invalid_argument);
+    return ScanQueryContext{std::make_error_code(std::errc::invalid_argument),
+                            std::move(ScannerDiagnosticsCollector), nullptr};
 
   (void)Instance->getMainModule();
 
-  return ScanQueryInstance{std::move(Instance), 
-                           scannerDiagnosticsCollector};
+  auto SerializedDiagnosticsOutputPath =
+      Instance->getInvocation()
+          .getSerializedDiagnosticsPathForAtMostOnePrimary();
+  std::unique_ptr<DiagnosticConsumer> SerailizedDiagnosticsConsumer;
+  if (!SerializedDiagnosticsOutputPath.empty()) {
+    SerailizedDiagnosticsConsumer =
+        swift::serialized_diagnostics::createThreadSafeConsumer(
+            SerializedDiagnosticsOutputPath, false);
+    Instance->addDiagnosticConsumer(SerailizedDiagnosticsConsumer.get());
+  }
+
+  return ScanQueryContext{std::move(Instance),
+                          std::move(ScannerDiagnosticsCollector),
+                          std::move(SerailizedDiagnosticsConsumer)};
 }
 
 } // namespace dependencies

lib/DependencyScan/ModuleDependencyScanner.cpp

@@ -237,7 +237,7 @@ ModuleDependencyScanningWorker::ModuleDependencyScanningWorker(
                       ScanASTContext.SourceMgr, *workerDiagnosticEngine));
 
   scanningASTDelegate = std::make_unique<InterfaceSubContextDelegateImpl>(
-      workerASTContext->SourceMgr, &workerASTContext->Diags,
+      workerASTContext->SourceMgr, workerDiagnosticEngine.get(),
       workerASTContext->SearchPathOpts, workerASTContext->LangOpts,
       workerASTContext->ClangImporterOpts, workerASTContext->CASOpts,
       workerCompilerInvocation->getFrontendOptions(),
@@ -314,14 +314,13 @@ ModuleDependencyScanningWorker::scanFilesystemForClangModuleDependency(
       clangScanningWorkingDirectoryPath, alreadySeenModules,
       lookupModuleOutput);
   if (!clangModuleDependencies) {
-    auto errorStr = toString(clangModuleDependencies.takeError());
-    // We ignore the "module 'foo' not found" error, the Swift dependency
-    // scanner will report such an error only if all of the module loaders
-    // fail as well.
-    if (errorStr.find("fatal error: module '" + moduleName.str().str() +
-                      "' not found") == std::string::npos)
-      workerASTContext->Diags.diagnose(
-          SourceLoc(), diag::clang_dependency_scan_error, errorStr);
+    llvm::handleAllErrors(clangModuleDependencies.takeError(), [this, &moduleName](
+                                                  const llvm::StringError &E) {
+          auto &message = E.getMessage();
+          if (message.find("fatal error: module '" + moduleName.str().str() +
+                            "' not found") == std::string::npos)
+            workerDiagnosticEngine->diagnose(SourceLoc(), diag::clang_dependency_scan_error, message);
+      });
     return std::nullopt;
   }
 
@@ -355,7 +354,7 @@ ModuleDependencyScanningWorker::scanHeaderDependenciesOfSwiftModule(
   auto clangModuleDependencies = scanHeaderDependencies();
   if (!clangModuleDependencies) {
     auto errorStr = toString(clangModuleDependencies.takeError());
-    workerASTContext->Diags.diagnose(
+    workerDiagnosticEngine->diagnose(
         SourceLoc(), diag::clang_header_dependency_scan_error, errorStr);
     return std::nullopt;
   }