[Diagnostics] Add a swift-syntax diagnostic style

The SwiftDiagnostics module within swift-syntax has a diagnostic
pretty-printer that does a nice rendering of the source code with
diagnostics placed inside gaps between the code lines.
Introduce another `-diagnostic-style` argument, `swift-syntax`,
to bridge from the pretty-printed C++ diagnostics over to the
swift-syntax diagnostics engine.

Author: DougGregor

include/swift/Basic/DiagnosticOptions.h

@@ -32,7 +32,7 @@ class DiagnosticOptions {
     VerifyAndApplyFixes
   } VerifyMode = NoVerify;
 
-  enum FormattingStyle { LLVM, Swift };
+  enum FormattingStyle { LLVM, Swift, SwiftSyntax };
 
   /// Indicates whether to allow diagnostics for \c <unknown> locations if
   /// \c VerifyMode is not \c NoVerify.

include/swift/Frontend/PrintingDiagnosticConsumer.h

@@ -45,6 +45,12 @@ class PrintingDiagnosticConsumer : public DiagnosticConsumer {
   SmallVector<std::string, 1> BufferedEducationalNotes;
   bool SuppressOutput = false;
 
+  /// swift-syntax rendering
+  void *queuedDiagnostics = nullptr;
+  void *queuedSourceFile = nullptr;
+  unsigned queuedDiagnosticsBufferID;
+  StringRef queuedBufferName;
+
 public:
   PrintingDiagnosticConsumer(llvm::raw_ostream &stream = llvm::errs());
   ~PrintingDiagnosticConsumer();

lib/ASTGen/Sources/ASTGen/Diagnostics.swift

@@ -122,3 +122,138 @@ func emitDiagnostic(
     )
   }
 }
+
+/// A set of queued diagnostics created by the C++ compiler and rendered
+/// via the swift-syntax renderer.
+struct QueuedDiagnostics {
+  /// The source file in which all of the diagnostics occur.
+  let sourceFile: SourceFileSyntax
+
+  /// The underlying buffer within the C++ SourceManager, which is used
+  /// for computations of source locations.
+  let buffer: UnsafeBufferPointer<UInt8>
+
+  /// The set of diagnostics.
+  fileprivate var diagnostics: [Diagnostic] = []
+
+  mutating func diagnose(_ diagnostic: Diagnostic) {
+    assert(diagnostic.node.root == sourceFile.root)
+    diagnostics.append(diagnostic)
+  }
+
+  func render() -> String {
+    return DiagnosticsFormatter.annotatedSource(
+      tree: sourceFile,
+      diags: diagnostics
+    )
+  }
+}
+
+/// Create a queued diagnostics structure in which we can
+@_cdecl("swift_ASTGen_createQueuedDiagnostics")
+public func createQueuedDiagnostics(
+  sourceFilePtr: UnsafeMutablePointer<UInt8>
+) -> UnsafeRawPointer {
+  return sourceFilePtr.withMemoryRebound(
+    to: ExportedSourceFile.self, capacity: 1
+  ) { sourceFile in
+    let ptr = UnsafeMutablePointer<QueuedDiagnostics>.allocate(capacity: 1)
+    ptr.initialize(to: .init(
+      sourceFile: sourceFile.pointee.syntax,
+      buffer: sourceFile.pointee.buffer)
+    )
+    return UnsafeRawPointer(ptr)
+  }
+}
+
+/// Destroy the queued diagnostics.
+@_cdecl("swift_ASTGen_destroyQueuedDiagnostics")
+public func destroyQueuedDiagnostics(
+  queuedDiagnosticsPtr: UnsafeMutablePointer<UInt8>
+) {
+  queuedDiagnosticsPtr.withMemoryRebound(to: QueuedDiagnostics.self, capacity: 1) { queuedDiagnostics in
+    queuedDiagnostics.deinitialize(count: 1)
+    queuedDiagnostics.deallocate()
+  }
+}
+
+/// Diagnostic message used for thrown errors.
+fileprivate struct SimpleDiagnostic: DiagnosticMessage {
+  let message: String
+
+  let severity: DiagnosticSeverity
+
+  var diagnosticID: MessageID {
+    .init(domain: "SwiftCompiler", id: "SimpleDiagnostic")
+  }
+}
+
+extension BridgedDiagnosticSeverity {
+  var asSeverity: DiagnosticSeverity {
+    switch self {
+    case .fatalError: return .error
+    case .error: return .error
+    case .warning: return .warning
+    case .remark: return .warning // FIXME
+    case .note: return .note
+    @unknown default: return .error
+    }
+  }
+}
+
+/// Add a new diagnostic to the queue.
+@_cdecl("swift_ASTGen_addQueuedDiagnostic")
+public func addQueuedDiagnostic(
+  queuedDiagnosticsPtr: UnsafeMutableRawPointer,
+  text: UnsafePointer<UInt8>,
+  textLength: Int,
+  severity: BridgedDiagnosticSeverity,
+  position: UnsafePointer<UInt8>
+) {
+  let queuedDiagnostics = queuedDiagnosticsPtr.bindMemory(
+    to: QueuedDiagnostics.self, capacity: 1
+  )
+
+  // Find the offset.
+  let buffer = queuedDiagnostics.pointee.buffer
+  let offset = position - buffer.baseAddress!
+  if offset < 0 || offset >= buffer.count {
+    return
+  }
+
+  // Find the token at that offset.
+  let sf = queuedDiagnostics.pointee.sourceFile
+  guard let token = sf.token(at: AbsolutePosition(utf8Offset: offset)) else {
+    return
+  }
+
+  let textBuffer = UnsafeBufferPointer(start: text, count: textLength)
+  let diagnostic = Diagnostic(
+    node: Syntax(token),
+    message: SimpleDiagnostic(
+      message: String(decoding: textBuffer, as: UTF8.self),
+      severity: severity.asSeverity
+    )
+  )
+
+  queuedDiagnostics.pointee.diagnose(diagnostic)
+}
+
+
+/// Render the queued diagnostics into a UTF-8 string.
+@_cdecl("swift_ASTGen_renderQueuedDiagnostics")
+public func renterQueuedDiagnostics(
+  queuedDiagnosticsPtr: UnsafeMutablePointer<UInt8>,
+  renderedPointer: UnsafeMutablePointer<UnsafePointer<UInt8>?>,
+  renderedLength: UnsafeMutablePointer<Int>
+) {
+  queuedDiagnosticsPtr.withMemoryRebound(to: QueuedDiagnostics.self, capacity: 1) { queuedDiagnostics in
+    let renderedStr = DiagnosticsFormatter.annotatedSource(
+      tree: queuedDiagnostics.pointee.sourceFile,
+      diags: queuedDiagnostics.pointee.diagnostics
+    )
+
+    (renderedPointer.pointee, renderedLength.pointee) =
+        allocateUTF8String(renderedStr)
+  }
+}

lib/ASTGen/Sources/ASTGen/Macros.swift

@@ -16,9 +16,12 @@ extension SyntaxProtocol {
     }
 
     // Otherwise, it must be one of our children.
-    return children(viewMode: .sourceAccurate).lazy.compactMap { child in
-      child.token(at: position)
-    }.first
+    for child in children(viewMode: .sourceAccurate) {
+      if let token = child.token(at: position) {
+        return token
+      }
+    }
+    fatalError("Children of syntax node do not cover all positions in it")
   }
 }
 
@@ -61,7 +64,7 @@ public func destroyMacro(
 }
 
 /// Allocate a copy of the given string as a UTF-8 string.
-private func allocateUTF8String(
+func allocateUTF8String(
   _ string: String,
   nullTerminated: Bool = false
 ) -> (UnsafePointer<UInt8>, Int) {

lib/Frontend/CompilerInvocation.cpp

@@ -1466,6 +1466,9 @@ static bool ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
       Opts.PrintedFormattingStyle = DiagnosticOptions::FormattingStyle::LLVM;
     } else if (contents == "swift") {
       Opts.PrintedFormattingStyle = DiagnosticOptions::FormattingStyle::Swift;
+    } else if (contents == "swift-syntax") {
+      Opts.PrintedFormattingStyle =
+          DiagnosticOptions::FormattingStyle::SwiftSyntax;
     } else {
       Diags.diagnose(SourceLoc(), diag::error_unsupported_option_argument,
                      arg->getOption().getPrefixedName(), arg->getValue());

lib/Frontend/PrintingDiagnosticConsumer.cpp

@@ -15,7 +15,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "swift/Frontend/PrintingDiagnosticConsumer.h"
+#include "swift/AST/CASTBridging.h"
 #include "swift/AST/DiagnosticEngine.h"
+#include "swift/AST/DiagnosticsCommon.h"
 #include "swift/Basic/LLVM.h"
 #include "swift/Basic/SourceManager.h"
 #include "swift/Markup/Markup.h"
@@ -31,6 +33,25 @@
 using namespace swift;
 using namespace swift::markup;
 
+extern "C" void *swift_ASTGen_createQueuedDiagnostics(void *sourceFile);
+extern "C" void swift_ASTGen_destroyQueuedDiagnostics(void *queued);
+extern "C" void swift_ASTGen_addQueuedDiagnostic(
+    void *queued,
+    const char* text, ptrdiff_t textLength,
+    BridgedDiagnosticSeverity severity,
+    const void *sourceLoc
+);
+extern "C" void swift_ASTGen_renderQueuedDiagnostics(
+    void *queued, char **outBuffer, ptrdiff_t *outBufferLength);
+
+// FIXME: Hack because we cannot easily get to the already-parsed source
+// file from here. Fix this egregious oversight!
+extern "C" void *swift_ASTGen_parseSourceFile(const char *buffer,
+                                              size_t bufferLength,
+                                              const char *moduleName,
+                                              const char *filename);
+extern "C" void swift_ASTGen_destroySourceFile(void *sourceFile);
+
 namespace {
   class ColoredStream : public raw_ostream {
     raw_ostream &Underlying;
@@ -134,7 +155,7 @@ namespace {
 
       void visitDocument(const Document *D) {
         for (const auto *Child : D->getChildren()) {
-          if (Child->getKind() == ASTNodeKind::Paragraph) {
+          if (Child->getKind() == markup::ASTNodeKind::Paragraph) {
             // Add a newline before top-level paragraphs
             printNewline();
           }
@@ -931,6 +952,43 @@ static void annotateSnippetWithInfo(SourceManager &SM,
   }
 }
 
+/// Enqueue a diagnostic with ASTGen's diagnostic rendering.
+static void enqueueDiagnostic(
+    void *queuedDiagnostics, const DiagnosticInfo &info, SourceManager &SM
+) {
+  llvm::SmallString<256> text;
+  {
+    llvm::raw_svector_ostream out(text);
+    DiagnosticEngine::formatDiagnosticText(out, info.FormatString,
+                                           info.FormatArgs);
+  }
+
+  BridgedDiagnosticSeverity severity;
+  switch (info.Kind) {
+  case DiagnosticKind::Error:
+    severity = BridgedDiagnosticSeverity::BridgedError;
+    break;
+
+  case DiagnosticKind::Warning:
+    severity = BridgedDiagnosticSeverity::BridgedWarning;
+    break;
+
+  case DiagnosticKind::Remark:
+    severity = BridgedDiagnosticSeverity::BridgedRemark;
+    break;
+
+  case DiagnosticKind::Note:
+    severity = BridgedDiagnosticSeverity::BridgedNote;
+    break;
+  }
+
+  swift_ASTGen_addQueuedDiagnostic(
+      queuedDiagnostics, text.data(), text.size(), severity,
+      info.Loc.getOpaquePointerValue());
+
+  // FIXME: Need a way to add highlights, Fix-Its, and so on.
+}
+
 // MARK: Main DiagnosticConsumer entrypoint.
 void PrintingDiagnosticConsumer::handleDiagnostic(SourceManager &SM,
                                                   const DiagnosticInfo &Info) {
@@ -965,6 +1023,43 @@ void PrintingDiagnosticConsumer::handleDiagnostic(SourceManager &SM,
     }
     break;
 
+  case DiagnosticOptions::FormattingStyle::SwiftSyntax: {
+    if (Info.Loc.isValid()) {
+      // Ignore "in macro expansion" diagnostics; we want to put them
+      // elsewhere.
+      // FIXME: We should render the "in macro expansion" information in
+      // some other manner. Not quite sure how at this point, though.
+      if (Info.ID == diag::in_macro_expansion.ID)
+        break;
+
+      // If there are no enqueued diagnostics, they are from a different
+      // buffer, flush any enqueued diagnostics and create a new set.
+      unsigned bufferID = SM.findBufferContainingLoc(Info.Loc);
+      if (!queuedDiagnostics || bufferID != queuedDiagnosticsBufferID) {
+        flush(/*includeTrailingBreak*/ true);
+
+        // FIXME: Go parse the source file again. This is an awful hack.
+        auto bufferContents = SM.getEntireTextForBuffer(bufferID);
+        queuedSourceFile = swift_ASTGen_parseSourceFile(
+            bufferContents.data(), bufferContents.size(),
+            "module", "file.swift");
+
+        queuedBufferName = SM.getDisplayNameForLoc(Info.Loc);
+        queuedDiagnostics =
+            swift_ASTGen_createQueuedDiagnostics(queuedSourceFile);
+        queuedDiagnosticsBufferID = bufferID;
+      }
+
+      enqueueDiagnostic(queuedDiagnostics, Info, SM);
+      break;
+    }
+
+    // Fall through to print using the LLVM style when there is no source
+    // location.
+    flush(/*includeTrailingBreak*/ false);
+    LLVM_FALLTHROUGH;
+  }
+
   case DiagnosticOptions::FormattingStyle::LLVM:
     printDiagnostic(SM, Info);
 
@@ -1000,6 +1095,25 @@ void PrintingDiagnosticConsumer::flush(bool includeTrailingBreak) {
     currentSnippet.reset();
   }
 
+  if (queuedDiagnostics) {
+    Stream << "=== " << queuedBufferName << " ===\n";
+
+    char *renderedString = nullptr;
+    ptrdiff_t renderedStringLen = 0;
+    swift_ASTGen_renderQueuedDiagnostics(
+        queuedDiagnostics, &renderedString, &renderedStringLen);
+    if (renderedString) {
+      Stream.write(renderedString, renderedStringLen);
+    }
+    swift_ASTGen_destroyQueuedDiagnostics(queuedDiagnostics);
+    swift_ASTGen_destroySourceFile(queuedSourceFile);
+    queuedDiagnostics = nullptr;
+    queuedSourceFile = nullptr;
+
+    if (includeTrailingBreak)
+      Stream << "\n";
+  }
+
   for (auto note : BufferedEducationalNotes) {
     printMarkdown(note, Stream, ForceColors);
     Stream << "\n";