Add support for emitting parseable-output "finished" message and per-primary messages for batched jobs

artemcm · artemcm · commit 23e374ce78bb · 2020-12-11T16:01:03.000-08:00
Inside swift-frontend
diff --git a/include/swift/Basic/ParseableOutput.h b/include/swift/Basic/ParseableOutput.h
@@ -20,31 +20,36 @@
 
 #include "swift/Basic/LLVM.h"
 #include "swift/Basic/TaskQueue.h"
-#include "swift/Frontend/Frontend.h"
 #include "swift/Driver/Job.h"
+#include "swift/Frontend/Frontend.h"
 
 namespace swift {
 
-
 namespace parseable_output {
 
 /// Emits a "began" message to the given stream, corresponding to a Driver Job.
 void emitBeganMessage(raw_ostream &os, const driver::Job &Cmd, int64_t Pid,
                       sys::TaskProcessInformation ProcInfo);
 
-/// Emits a "began" message to the given stream, corresponding to a given Frontend
-/// Compiler Invocation.
-void emitBeganMessage(raw_ostream &os,
-                      const CompilerInvocation &Invocation,
-                      ArrayRef<const char *> Args,
-                      int64_t Pid,
+/// Emits a "began" message to the given stream, corresponding to a given
+/// Frontend Compiler Invocation.
+void emitBeganMessage(raw_ostream &os, const CompilerInvocation &Invocation,
+                      ArrayRef<const char *> Args, int64_t Pid,
                       sys::TaskProcessInformation ProcInfo);
 
 /// Emits a "finished" message to the given stream.
 void emitFinishedMessage(raw_ostream &os, const driver::Job &Cmd, int64_t Pid,
                          int ExitStatus, StringRef Output,
                          sys::TaskProcessInformation ProcInfo);
 
+/// Emits a "finished" message to the given stream corresponding to a given
+/// Frontend Compiler Invocation.
+void emitFinishedMessage(
+    raw_ostream &os, const CompilerInvocation &Invocation, int64_t Pid,
+    int ExitStatus,
+    const llvm::StringMap<std::vector<std::string>> &FileSpecificDiagnostics,
+    sys::TaskProcessInformation ProcInfo);
+
 /// Emits a "signalled" message to the given stream.
 void emitSignalledMessage(raw_ostream &os, const driver::Job &Cmd, int64_t Pid,
                           StringRef ErrorMsg, StringRef Output,
diff --git a/include/swift/Frontend/AccumulatingDiagnosticConsumer.h b/include/swift/Frontend/AccumulatingDiagnosticConsumer.h
@@ -0,0 +1,51 @@
+//===- AccumulatingDiagnosticConsumer.h - Collect Text Diagnostics  C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the AccumulatingDiagnosticConsumer class, which collects
+//  all emitted diagnostics into an externally-owned collection.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_ACCUMULATINGDIAGNOSTICCONSUMER_H
+#define SWIFT_ACCUMULATINGDIAGNOSTICCONSUMER_H
+
+#include "swift/AST/DiagnosticConsumer.h"
+#include "swift/Basic/DiagnosticOptions.h"
+#include "swift/Basic/LLVM.h"
+
+#include <string>
+#include <sstream>
+
+namespace swift {
+/// Diagnostic consumer that simply collects all emitted diagnostics into the provided
+/// collection.
+class AccumulatingFileDiagnosticConsumer : public DiagnosticConsumer {
+  std::vector<std::string> &Diagnostics;
+public:
+  AccumulatingFileDiagnosticConsumer(std::vector<std::string> &DiagBuffer)
+    : Diagnostics(DiagBuffer) {}
+
+private:
+  void handleDiagnostic(SourceManager &SM,
+                        const DiagnosticInfo &Info) override {
+    std::string DiagMsg;
+    llvm::raw_string_ostream DiagOS(DiagMsg);
+    DiagnosticEngine::formatDiagnosticText(DiagOS, Info.FormatString,
+                                           Info.FormatArgs);
+    auto LC = SM.getPresumedLineAndColumnForLoc(Info.Loc);
+    std::ostringstream StrOS;
+    StrOS << LC.first << ", " << LC.second << ": " << DiagOS.str();
+    Diagnostics.push_back(StrOS.str());
+  }
+};
+}
+#endif
diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h
@@ -184,6 +184,11 @@ class FrontendOptions {
   /// entity.
   bool ProfileEntities = false;
 
+  /// Emit parseable-output directly from the frontend, instead of relying
+  /// the driver to emit it. This is used in context where frontend jobs are executed by
+  /// clients other than the driver.
+  bool FrontendParseableOutput = false;
+
   /// Indicates whether or not an import statement can pick up a Swift source
   /// file (as opposed to a module file).
   bool EnableSourceImport = false;
diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td
@@ -30,6 +30,8 @@ def output_filelist : Separate<["-"], "output-filelist">,
 def supplementary_output_file_map : Separate<["-"], "supplementary-output-file-map">,
   HelpText<"Specify supplementary outputs in a file rather than on the command line">;
 
+def frontend_parseable_output : Flag<["-"], "frontend-parseable-output">,
+  HelpText<"Emit textual output in a parseable format">;
 
 def emit_module_doc : Flag<["-"], "emit-module-doc">,
   HelpText<"Emit a module documentation file based on documentation "
diff --git a/lib/Basic/ParseableOutput.cpp b/lib/Basic/ParseableOutput.cpp
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "swift/Basic/ParseableOutput.h"
-
 #include "swift/Basic/FileTypes.h"
 #include "swift/Basic/JSONSerialization.h"
 #include "swift/Basic/TaskQueue.h"
@@ -21,6 +20,8 @@
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 
+#include <sstream>
+
 using namespace swift::parseable_output;
 using namespace swift::driver;
 using namespace swift::sys;
@@ -221,17 +222,9 @@ class DetailedInvocationBasedMessage : public InvocationBasedMessage {
 public:
   DetailedInvocationBasedMessage(StringRef Kind,
                                  const CompilerInvocation &Invocation,
+                                 const InputFile &PrimaryInput,
                                  ArrayRef<const char *> Args)
       : InvocationBasedMessage(Kind, Invocation), Args(Args) {
-    const auto &IO = Invocation.getFrontendOptions().InputsAndOutputs;
-    // This frontend invocation correpsonds to a BatchJob:
-    // A container of multiple StandardCompile Jobs.
-    if (IO.hasMultiplePrimaryInputs()) {
-      llvm::dbgs() << "Batch JOB.\n";
-    } else {
-      llvm::dbgs() << "Single JOB.\n";
-    }
-
     // Command line and arguments
     Executable = Invocation.getFrontendOptions().MainExecutablePath;
     CommandLine += Executable;
@@ -240,30 +233,23 @@ class DetailedInvocationBasedMessage : public InvocationBasedMessage {
       CommandLine += std::string(" ") + A;
     }
 
-    // Primary Inputs
-    IO.forEachPrimaryInput([&](const InputFile &input) -> bool {
-      Inputs.push_back(CommandInput(input.getFileName()));
-      return false;
-    });
+    // Primary Input only
+    Inputs.push_back(CommandInput(PrimaryInput.getFileName()));
 
-    // Outputs
-    IO.forEachOutputFilename([&](const StringRef &output) {
-      Outputs.push_back(OutputPair(file_types::lookupTypeForExtension(
-                                       llvm::sys::path::extension(output)),
-                                   output.str()));
-    });
+    // Output for this Primary
+    auto OutputFile = PrimaryInput.outputFilename();
+    Outputs.push_back(OutputPair(file_types::lookupTypeForExtension(
+                                     llvm::sys::path::extension(OutputFile)),
+                                 OutputFile));
 
     // Supplementary outputs
-    IO.forEachPrimaryInput([&](const InputFile &input) -> bool {
-      const auto &primarySpecificFiles = input.getPrimarySpecificPaths();
-      const auto &supplementaryOutputPaths =
-          primarySpecificFiles.SupplementaryOutputs;
-      supplementaryOutputPaths.forEachSetOutput([&](const std::string &output) {
-        Outputs.push_back(OutputPair(file_types::lookupTypeForExtension(
-                                         llvm::sys::path::extension(output)),
-                                     output));
-      });
-      return false;
+    const auto &primarySpecificFiles = PrimaryInput.getPrimarySpecificPaths();
+    const auto &supplementaryOutputPaths =
+        primarySpecificFiles.SupplementaryOutputs;
+    supplementaryOutputPaths.forEachSetOutput([&](const std::string &output) {
+      Outputs.push_back(OutputPair(file_types::lookupTypeForExtension(
+                                       llvm::sys::path::extension(output)),
+                                   output));
     });
   }
 
@@ -290,22 +276,6 @@ class TaskBasedMessage : public CommandBasedMessage {
   }
 };
 
-class FinishedInvocationBasedMessage : public InvocationBasedMessage {
-  int64_t Pid;
-
-public:
-  FinishedInvocationBasedMessage(StringRef Kind,
-                                 const CompilerInvocation &Invocation,
-                                 int64_t Pid,
-                                 StringRef Output,
-                                 int ExitStatus)
-  : InvocationBasedMessage(Kind, Invocation), Pid(Pid) {}
-  void provideMapping(swift::json::Output &out) override {
-    InvocationBasedMessage::provideMapping(out);
-    out.mapRequired("pid", Pid);
-  }
-};
-
 class BeganCommandMessage : public DetailedCommandBasedMessage {
   int64_t Pid;
   TaskProcessInformation ProcInfo;
@@ -329,10 +299,11 @@ class BeganInvocationMessage : public DetailedInvocationBasedMessage {
 
 public:
   BeganInvocationMessage(const CompilerInvocation &Invocation,
+                         const InputFile &PrimaryInput,
                          ArrayRef<const char *> Args, int64_t Pid,
                          TaskProcessInformation ProcInfo)
-      : DetailedInvocationBasedMessage("began", Invocation, Args), Pid(Pid),
-        ProcInfo(ProcInfo) {}
+      : DetailedInvocationBasedMessage("began", Invocation, PrimaryInput, Args),
+        Pid(Pid), ProcInfo(ProcInfo) {}
 
   void provideMapping(swift::json::Output &out) override {
     DetailedInvocationBasedMessage::provideMapping(out);
@@ -341,6 +312,27 @@ class BeganInvocationMessage : public DetailedInvocationBasedMessage {
   }
 };
 
+class FinishedInvocationMessage : public InvocationBasedMessage {
+  int64_t Pid;
+  int ExitStatus;
+  std::string Output;
+  TaskProcessInformation ProcInfo;
+
+public:
+  FinishedInvocationMessage(const CompilerInvocation &Invocation, int64_t Pid,
+                            StringRef Output, int ExitStatus,
+                            TaskProcessInformation ProcInfo)
+      : InvocationBasedMessage("finished", Invocation), Pid(Pid),
+        ExitStatus(ExitStatus), Output(Output), ProcInfo(ProcInfo) {}
+  void provideMapping(swift::json::Output &out) override {
+    InvocationBasedMessage::provideMapping(out);
+    out.mapOptional("output", Output, std::string());
+    out.mapRequired("process", ProcInfo);
+    out.mapRequired("pid", Pid);
+    out.mapRequired("exit-status", ExitStatus);
+  }
+};
+
 class TaskOutputMessage : public TaskBasedMessage {
   std::string Output;
   TaskProcessInformation ProcInfo;
@@ -362,7 +354,7 @@ class FinishedCommandMessage : public TaskOutputMessage {
 
 public:
   FinishedCommandMessage(const driver::Job &Cmd, int64_t Pid, StringRef Output,
-                  TaskProcessInformation ProcInfo, int ExitStatus)
+                         TaskProcessInformation ProcInfo, int ExitStatus)
       : TaskOutputMessage("finished", Cmd, Pid, Output, ProcInfo),
         ExitStatus(ExitStatus) {}
 
@@ -418,27 +410,58 @@ static void emitMessage(raw_ostream &os, Message &msg) {
   os << JSONString << '\n';
 }
 
-void parseable_output::emitBeganMessage(
-    raw_ostream &os, const driver::Job &Cmd, int64_t Pid,
-    TaskProcessInformation ProcInfo) {
+void parseable_output::emitBeganMessage(raw_ostream &os, const driver::Job &Cmd,
+                                        int64_t Pid,
+                                        TaskProcessInformation ProcInfo) {
   BeganCommandMessage msg(Cmd, Pid, ProcInfo);
   emitMessage(os, msg);
 }
 
-void parseable_output::emitBeganMessage(
-    raw_ostream &os, const CompilerInvocation &Invocation,
-    ArrayRef<const char *> Args, int64_t Pid, TaskProcessInformation ProcInfo) {
-  BeganInvocationMessage msg(Invocation, Args, Pid, ProcInfo);
-  emitMessage(os, msg);
+void parseable_output::emitBeganMessage(raw_ostream &os,
+                                        const CompilerInvocation &Invocation,
+                                        ArrayRef<const char *> Args,
+                                        int64_t Pid,
+                                        TaskProcessInformation ProcInfo) {
+  const auto &IO = Invocation.getFrontendOptions().InputsAndOutputs;
+  IO.forEachPrimaryInput([&](const InputFile &Input) -> bool {
+    BeganInvocationMessage msg(Invocation, Input, Args, Pid, ProcInfo);
+    emitMessage(os, msg);
+    return false;
+  });
 }
 
-void parseable_output::emitFinishedMessage(
-    raw_ostream &os, const driver::Job &Cmd, int64_t Pid, int ExitStatus,
-    StringRef Output, TaskProcessInformation ProcInfo) {
+void parseable_output::emitFinishedMessage(raw_ostream &os,
+                                           const driver::Job &Cmd, int64_t Pid,
+                                           int ExitStatus, StringRef Output,
+                                           TaskProcessInformation ProcInfo) {
   FinishedCommandMessage msg(Cmd, Pid, Output, ProcInfo, ExitStatus);
   emitMessage(os, msg);
 }
 
+void parseable_output::emitFinishedMessage(
+    raw_ostream &os, const CompilerInvocation &Invocation, int64_t Pid,
+    int ExitStatus,
+    const llvm::StringMap<std::vector<std::string>> &FileSpecificDiagnostics,
+    TaskProcessInformation ProcInfo) {
+  const auto &IO = Invocation.getFrontendOptions().InputsAndOutputs;
+  IO.forEachPrimaryInput([&](const InputFile &Input) -> bool {
+    assert(FileSpecificDiagnostics.count(Input.getFileName()) != 0 &&
+           "Expected diagnostic collection for input.");
+
+    // Join all diagnostics produced for this file into a single output.
+    auto PrimaryDiags = FileSpecificDiagnostics.lookup(Input.getFileName());
+    const char* const Delim = "\n";
+    std::ostringstream JoinedDiags;
+    std::copy(PrimaryDiags.begin(), PrimaryDiags.end(),
+              std::ostream_iterator<std::string>(JoinedDiags, Delim));
+    FinishedInvocationMessage msg(Invocation, Pid,
+                                  JoinedDiags.str(),
+                                  ExitStatus, ProcInfo);
+    emitMessage(os, msg);
+    return false;
+  });
+}
+
 void parseable_output::emitSignalledMessage(
     raw_ostream &os, const driver::Job &Cmd, int64_t Pid, StringRef ErrorMsg,
     StringRef Output, Optional<int> Signal, TaskProcessInformation ProcInfo) {
@@ -447,7 +470,7 @@ void parseable_output::emitSignalledMessage(
 }
 
 void parseable_output::emitSkippedMessage(raw_ostream &os,
-                                                   const driver::Job &Cmd) {
+                                          const driver::Job &Cmd) {
   SkippedMessage msg(Cmd);
   emitMessage(os, msg);
 }
diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp
@@ -77,6 +77,7 @@ bool ArgsToFrontendOptionsConverter::convert(
   Opts.EnableTesting |= Args.hasArg(OPT_enable_testing);
   Opts.EnablePrivateImports |= Args.hasArg(OPT_enable_private_imports);
   Opts.EnableLibraryEvolution |= Args.hasArg(OPT_enable_library_evolution);
+  Opts.FrontendParseableOutput |= Args.hasArg(OPT_frontend_parseable_output);
 
   // FIXME: Remove this flag
   Opts.EnableLibraryEvolution |= Args.hasArg(OPT_enable_resilience);
diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp
