Merge pull request github#11101 from github/alexdenisov/extractor-errors

Swift: extract diagnostics

Author: AlexDenisov

swift/extractor/BUILD.bazel

@@ -10,6 +10,7 @@ swift_cc_binary(
     visibility = ["//swift:__pkg__"],
     deps = [
         "//swift/extractor/infra",
+        "//swift/extractor/invocation",
         "//swift/extractor/remapping",
         "//swift/extractor/translators",
         "//swift/third_party/swift-llvm-support",

swift/extractor/infra/SwiftDispatcher.h

@@ -10,7 +10,7 @@
 #include "swift/extractor/trap/TrapDomain.h"
 #include "swift/extractor/infra/SwiftTagTraits.h"
 #include "swift/extractor/trap/generated/TrapClasses.h"
-#include "swift/extractor/infra/file/PathHash.h"
+#include "swift/extractor/infra/SwiftLocationExtractor.h"
 
 namespace codeql {
 
@@ -29,8 +29,7 @@ class SwiftDispatcher {
                                const swift::Expr*,
                                const swift::Pattern*,
                                const swift::TypeRepr*,
-                               const swift::TypeBase*,
-                               std::filesystem::path>;
+                               const swift::TypeBase*>;
 
   template <typename E>
   static constexpr bool IsStorable = std::is_constructible_v<Store::Handle, const E&>;
@@ -48,10 +47,11 @@ class SwiftDispatcher {
       : sourceManager{sourceManager},
         trap{trap},
         currentModule{currentModule},
-        currentPrimarySourceFile{currentPrimarySourceFile} {
+        currentPrimarySourceFile{currentPrimarySourceFile},
+        locationExtractor(trap) {
     if (currentPrimarySourceFile) {
       // we make sure the file is in the trap output even if the source is empty
-      fetchLabel(getFilePath(currentPrimarySourceFile->getFilename()));
+      locationExtractor.emitFile(currentPrimarySourceFile->getFilename());
     }
   }
 
@@ -325,20 +325,7 @@ class SwiftDispatcher {
   void attachLocation(swift::SourceLoc start,
                       swift::SourceLoc end,
                       TrapLabel<LocatableTag> locatableLabel) {
-    if (!start.isValid() || !end.isValid()) {
-      // invalid locations seem to come from entities synthesized by the compiler
-      return;
-    }
-    auto file = getFilePath(sourceManager.getDisplayNameForLoc(start));
-    DbLocation entry{{}};
-    entry.file = fetchLabel(file);
-    std::tie(entry.start_line, entry.start_column) = sourceManager.getLineAndColumnInBuffer(start);
-    std::tie(entry.end_line, entry.end_column) = sourceManager.getLineAndColumnInBuffer(end);
-    entry.id = trap.createLabel<DbLocationTag>('{', entry.file, "}:", entry.start_line, ':',
-                                               entry.start_column, ':', entry.end_line, ':',
-                                               entry.end_column);
-    emit(entry);
-    emit(LocatableLocationsTrap{locatableLabel, entry.id});
+    locationExtractor.attachLocation(sourceManager, start, end, locatableLabel);
   }
 
   template <typename Tag, typename... Ts>
@@ -391,19 +378,14 @@ class SwiftDispatcher {
   virtual void visit(const swift::TypeRepr* typeRepr, swift::Type type) = 0;
   virtual void visit(const swift::TypeBase* type) = 0;
 
-  void visit(const std::filesystem::path& file) {
-    auto entry = createEntry(file, file.string());
-    entry.name = file.string();
-    emit(entry);
-  }
-
   const swift::SourceManager& sourceManager;
   TrapDomain& trap;
   Store store;
   Store::Handle waitingForNewLabel{std::monostate{}};
   swift::ModuleDecl& currentModule;
   swift::SourceFile* currentPrimarySourceFile;
   std::unordered_set<swift::ModuleDecl*> encounteredModules;
+  SwiftLocationExtractor locationExtractor;
 };
 
 }  // namespace codeql

swift/extractor/infra/SwiftLocationExtractor.cpp

@@ -0,0 +1,59 @@
+#include <swift/AST/SourceFile.h>
+#include <swift/Basic/SourceManager.h>
+
+#include "swift/extractor/trap/TrapDomain.h"
+#include "swift/extractor/trap/generated/TrapEntries.h"
+#include "swift/extractor/trap/generated/TrapClasses.h"
+#include "swift/extractor/infra/SwiftLocationExtractor.h"
+
+using namespace codeql;
+
+static std::filesystem::path getFilePath(std::string_view path) {
+  // TODO: this needs more testing
+  // TODO: check canonicalization of names on a case insensitive filesystems
+  // TODO: make symlink resolution conditional on CODEQL_PRESERVE_SYMLINKS=true
+  std::error_code ec;
+  auto ret = std::filesystem::canonical(path, ec);
+  if (ec) {
+    std::cerr << "Cannot get real path: " << std::quoted(path) << ": " << ec.message() << "\n";
+    return {};
+  }
+  return ret;
+}
+
+void SwiftLocationExtractor::attachLocation(const swift::SourceManager& sourceManager,
+                                            swift::SourceLoc start,
+                                            swift::SourceLoc end,
+                                            TrapLabel<LocatableTag> locatableLabel) {
+  if (!start.isValid() || !end.isValid()) {
+    // invalid locations seem to come from entities synthesized by the compiler
+    return;
+  }
+  auto file = getFilePath(sourceManager.getDisplayNameForLoc(start));
+  DbLocation entry{{}};
+  entry.file = fetchFileLabel(file);
+  std::tie(entry.start_line, entry.start_column) = sourceManager.getLineAndColumnInBuffer(start);
+  std::tie(entry.end_line, entry.end_column) = sourceManager.getLineAndColumnInBuffer(end);
+  entry.id = trap.createLabel<DbLocationTag>('{', entry.file, "}:", entry.start_line, ':',
+                                             entry.start_column, ':', entry.end_line, ':',
+                                             entry.end_column);
+  trap.emit(entry);
+  trap.emit(LocatableLocationsTrap{locatableLabel, entry.id});
+}
+
+void SwiftLocationExtractor::emitFile(llvm::StringRef path) {
+  fetchFileLabel(getFilePath(path));
+}
+
+TrapLabel<FileTag> SwiftLocationExtractor::fetchFileLabel(const std::filesystem::path& file) {
+  if (store.count(file)) {
+    return store[file];
+  }
+
+  DbFile entry({});
+  entry.id = trap.createLabel<DbFileTag>(file.string());
+  entry.name = file.string();
+  trap.emit(entry);
+  store[file] = entry.id;
+  return entry.id;
+}

swift/extractor/infra/SwiftLocationExtractor.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <swift/Basic/SourceManager.h>
+#include <unordered_map>
+#include <filesystem>
+
+#include "swift/extractor/trap/generated/TrapEntries.h"
+#include "swift/extractor/infra/file/PathHash.h"
+
+namespace codeql {
+
+class TrapDomain;
+
+class SwiftLocationExtractor {
+ public:
+  explicit SwiftLocationExtractor(TrapDomain& trap) : trap(trap) {}
+
+  void attachLocation(const swift::SourceManager& sourceManager,
+                      swift::SourceLoc start,
+                      swift::SourceLoc end,
+                      TrapLabel<LocatableTag> locatableLabel);
+
+  void emitFile(llvm::StringRef path);
+
+ private:
+  TrapLabel<FileTag> fetchFileLabel(const std::filesystem::path& file);
+  TrapDomain& trap;
+  std::unordered_map<std::filesystem::path, TrapLabel<FileTag>> store;
+};
+
+}  // namespace codeql

swift/extractor/invocation/BUILD.bazel

@@ -0,0 +1,11 @@
+load("//swift:rules.bzl", "swift_cc_library")
+
+swift_cc_library(
+    name = "invocation",
+    srcs = glob(["*.cpp"]),
+    hdrs = glob(["*.h"]),
+    visibility = ["//swift:__subpackages__"],
+    deps = [
+        "//swift/extractor/infra",
+    ],
+)

swift/extractor/invocation/SwiftDiagnosticsConsumer.cpp

@@ -0,0 +1,44 @@
+#include "swift/extractor/invocation/SwiftDiagnosticsConsumer.h"
+#include "swift/extractor/trap/generated/TrapEntries.h"
+#include "swift/extractor/trap/TrapDomain.h"
+
+#include <swift/AST/DiagnosticEngine.h>
+#include <swift/Basic/SourceManager.h>
+#include <llvm/ADT/SmallString.h>
+#include <llvm/Support/raw_ostream.h>
+#include <string>
+
+using namespace codeql;
+
+static int diagnosticsKind(const swift::DiagnosticInfo& diagInfo) {
+  switch (diagInfo.Kind) {
+    case swift::DiagnosticKind::Error:
+      return 1;
+    case swift::DiagnosticKind::Warning:
+      return 2;
+    case swift::DiagnosticKind::Note:
+      return 3;
+    case swift::DiagnosticKind::Remark:
+      return 4;
+  }
+  return 0;
+}
+
+void SwiftDiagnosticsConsumer::handleDiagnostic(swift::SourceManager& sourceManager,
+                                                const swift::DiagnosticInfo& diagInfo) {
+  auto message = getDiagMessage(sourceManager, diagInfo);
+  DiagnosticsTrap diag{};
+  diag.id = trap.createLabel<DiagnosticsTag>();
+  diag.kind = diagnosticsKind(diagInfo);
+  diag.text = message;
+  trap.emit(diag);
+  locationExtractor.attachLocation(sourceManager, diagInfo.Loc, diagInfo.Loc, diag.id);
+}
+
+std::string SwiftDiagnosticsConsumer::getDiagMessage(swift::SourceManager& sourceManager,
+                                                     const swift::DiagnosticInfo& diagInfo) {
+  llvm::SmallString<256> text;
+  llvm::raw_svector_ostream out(text);
+  swift::DiagnosticEngine::formatDiagnosticText(out, diagInfo.FormatString, diagInfo.FormatArgs);
+  return text.str().str();
+}

swift/extractor/invocation/SwiftDiagnosticsConsumer.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <swift/AST/DiagnosticConsumer.h>
+#include "swift/extractor/infra/SwiftLocationExtractor.h"
+
+namespace codeql {
+
+class TrapDomain;
+
+class SwiftDiagnosticsConsumer : public swift::DiagnosticConsumer {
+ public:
+  explicit SwiftDiagnosticsConsumer(TrapDomain& targetFile)
+      : trap(targetFile), locationExtractor(targetFile) {}
+  void handleDiagnostic(swift::SourceManager& sourceManager,
+                        const swift::DiagnosticInfo& diagInfo) override;
+
+ private:
+  static std::string getDiagMessage(swift::SourceManager& sourceManager,
+                                    const swift::DiagnosticInfo& diagInfo);
+  TrapDomain& trap;
+  SwiftLocationExtractor locationExtractor;
+};
+
+}  // namespace codeql

swift/extractor/main.cpp

@@ -1,12 +1,11 @@
 #include <fstream>
 #include <iomanip>
 #include <stdlib.h>
-#include <unordered_set>
 #include <vector>
-#include <string>
 #include <iostream>
 #include <regex>
 #include <unistd.h>
+#include <chrono>
 
 #include <swift/Basic/LLVMInitialize.h>
 #include <swift/FrontendTool/FrontendTool.h>
@@ -15,6 +14,8 @@
 #include "swift/extractor/TargetTrapFile.h"
 #include "swift/extractor/remapping/SwiftOutputRewrite.h"
 #include "swift/extractor/remapping/SwiftOpenInterception.h"
+#include "swift/extractor/invocation/SwiftDiagnosticsConsumer.h"
+#include "swift/extractor/trap/TrapDomain.h"
 
 using namespace std::string_literals;
 
@@ -23,14 +24,21 @@ using namespace std::string_literals;
 // semantic analysis
 class Observer : public swift::FrontendObserver {
  public:
-  explicit Observer(const codeql::SwiftExtractorConfiguration& config) : config{config} {}
+  explicit Observer(const codeql::SwiftExtractorConfiguration& config,
+                    codeql::SwiftDiagnosticsConsumer& diagConsumer)
+      : config{config}, diagConsumer{diagConsumer} {}
+
+  void configuredCompiler(swift::CompilerInstance& instance) override {
+    instance.addDiagnosticConsumer(&diagConsumer);
+  }
 
   void performedSemanticAnalysis(swift::CompilerInstance& compiler) override {
     codeql::extractSwiftFiles(config, compiler);
   }
 
  private:
   const codeql::SwiftExtractorConfiguration& config;
+  codeql::SwiftDiagnosticsConsumer& diagConsumer;
 };
 
 static std::string getenv_or(const char* envvar, const std::string& def) {
@@ -96,6 +104,20 @@ static void checkWhetherToRunUnderTool(int argc, char* const* argv) {
   execvp(args[0], args.data());
 }
 
+// Creates a target file that should store per-invocation info, e.g. compilation args,
+// compilations, diagnostics, etc.
+codeql::TargetFile invocationTargetFile(codeql::SwiftExtractorConfiguration& configuration) {
+  auto timestamp = std::chrono::system_clock::now().time_since_epoch().count();
+  auto filename = std::to_string(timestamp) + '-' + std::to_string(getpid());
+  auto target = std::filesystem::path("invocations") / std::filesystem::path(filename);
+  auto maybeFile = codeql::createTargetTrapFile(configuration, target);
+  if (!maybeFile) {
+    std::cerr << "Cannot create invocation trap file: " << target << "\n";
+    abort();
+  }
+  return std::move(maybeFile.value());
+}
+
 int main(int argc, char** argv) {
   checkWhetherToRunUnderTool(argc, argv);
 
@@ -131,7 +153,10 @@ int main(int argc, char** argv) {
     args.push_back(arg.c_str());
   }
 
-  Observer observer(configuration);
+  auto invocationTrapFile = invocationTargetFile(configuration);
+  codeql::TrapDomain invocationDomain(invocationTrapFile);
+  codeql::SwiftDiagnosticsConsumer diagConsumer(invocationDomain);
+  Observer observer(configuration, diagConsumer);
   int frontend_rc = swift::performFrontend(args, "swift-extractor", (void*)main, &observer);
 
   codeql::finalizeRemapping(remapping);

swift/ql/lib/codeql/swift/elements.qll

@@ -4,6 +4,7 @@ import codeql.swift.elements.Callable
 import codeql.swift.elements.Comment
 import codeql.swift.elements.DbFile
 import codeql.swift.elements.DbLocation
+import codeql.swift.elements.Diagnostics
 import codeql.swift.elements.Element
 import codeql.swift.elements.File
 import codeql.swift.elements.Locatable

swift/ql/lib/codeql/swift/elements/Diagnostics.qll

@@ -0,0 +1,5 @@
+private import codeql.swift.generated.Diagnostics
+
+class Diagnostics extends Generated::Diagnostics {
+  override string toString() { result = getText() }
+}