Swift: teach autobuilder about SPM, CocoaPods, and Carthage

Author: AlexDenisov

swift/tools/autobuild.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
 if [[ "$OSTYPE" == "darwin"* ]]; then
+  export CODEQL_SWIFT_CARTHAGE_EXEC=`which carthage`
+  export CODEQL_SWIFT_POD_EXEC=`which pod`
   exec "${CODEQL_EXTRACTOR_SWIFT_ROOT}/tools/${CODEQL_PLATFORM}/xcode-autobuilder"
 else
   exec "${CODEQL_EXTRACTOR_SWIFT_ROOT}/tools/${CODEQL_PLATFORM}/autobuilder-incompatible-os"

swift/xcode-autobuilder/XcodeBuildRunner.cpp

@@ -51,7 +51,22 @@ static bool exec(const std::vector<std::string>& argv) {
   return true;
 }
 
-void buildTarget(Target& target, bool dryRun) {
+static bool run_build_command(const std::vector<std::string>& argv, bool dryRun) {
+  if (dryRun) {
+    std::cout << absl::StrJoin(argv, " ") << "\n";
+  } else {
+    if (!exec(argv)) {
+      DIAGNOSE_ERROR(buildCommandFailed,
+                     "`autobuild` failed to run the build command:\n\n```\n{}\n```",
+                     absl::StrJoin(argv, " "));
+      codeql::Log::flush();
+      return false;
+    }
+  }
+  return true;
+}
+
+bool buildXcodeTarget(XcodeTarget& target, bool dryRun) {
   std::vector<std::string> argv({"/usr/bin/xcodebuild", "build"});
   if (!target.workspace.empty()) {
     argv.push_back("-workspace");
@@ -65,16 +80,40 @@ void buildTarget(Target& target, bool dryRun) {
   argv.push_back(target.name);
   argv.push_back("CODE_SIGNING_REQUIRED=NO");
   argv.push_back("CODE_SIGNING_ALLOWED=NO");
+  return run_build_command(argv, dryRun);
+}
 
-  if (dryRun) {
-    std::cout << absl::StrJoin(argv, " ") << "\n";
-  } else {
-    if (!exec(argv)) {
-      DIAGNOSE_ERROR(buildCommandFailed,
-                     "`autobuild` failed to run the detected build command:\n\n```\n{}\n```",
-                     absl::StrJoin(argv, " "));
-      codeql::Log::flush();
-      exit(1);
+bool buildSwiftPackage(const std::filesystem::path& packageFile, bool dryRun) {
+  std::vector<std::string> argv(
+      {"/usr/bin/swift", "build", "--package-path", packageFile.parent_path()});
+  return run_build_command(argv, dryRun);
+}
+
+static void pod_install(const std::string& pod, const std::filesystem::path& podfile, bool dryRun) {
+  std::vector<std::string> argv(
+      {pod, "install", "--project-directory=" + podfile.parent_path().string()});
+  run_build_command(argv, dryRun);
+}
+
+static void carthage_install(const std::string& carthage,
+                             const std::filesystem::path& podfile,
+                             bool dryRun) {
+  std::vector<std::string> argv(
+      {carthage, "bootstrap", "--project-directory", podfile.parent_path()});
+  run_build_command(argv, dryRun);
+}
+
+void installDependencies(ProjectStructure& target, bool dryRun) {
+  auto pod = std::string(getenv("CODEQL_SWIFT_POD_EXEC"));
+  auto carthage = std::string(getenv("CODEQL_SWIFT_CARTHAGE_EXEC"));
+  if (!pod.empty() && !target.podfiles.empty()) {
+    for (auto& podfile : target.podfiles) {
+      pod_install(pod, podfile, dryRun);
+    }
+  }
+  if (!carthage.empty() && !target.cartfiles.empty()) {
+    for (auto& cartfile : target.cartfiles) {
+      carthage_install(carthage, cartfile, dryRun);
     }
   }
 }

swift/xcode-autobuilder/XcodeBuildRunner.h

@@ -1,5 +1,9 @@
 #pragma once
 
 #include "swift/xcode-autobuilder/XcodeTarget.h"
+#include "swift/xcode-autobuilder/XcodeProjectParser.h"
+#include <filesystem>
 
-void buildTarget(Target& target, bool dryRun);
+void installDependencies(ProjectStructure& target, bool dryRun);
+bool buildXcodeTarget(XcodeTarget& target, bool dryRun);
+bool buildSwiftPackage(const std::filesystem::path& packageFile, bool dryRun);

swift/xcode-autobuilder/XcodeProjectParser.cpp

@@ -198,14 +198,30 @@ static std::unordered_map<std::string, TargetData> mapTargetsToWorkspace(
   return targetMapping;
 }
 
-static std::vector<fs::path> collectFiles(const std::string& workingDir) {
+struct ProjectFiles {
+  std::vector<fs::path> xcodeFiles;
+  std::vector<fs::path> packageFiles;
+  std::vector<fs::path> podfiles;
+  std::vector<fs::path> cartfiles;
+};
+
+static ProjectFiles scanWorkingDir(const std::string& workingDir) {
+  ProjectFiles structure;
   fs::path workDir(workingDir);
   std::vector<fs::path> files;
   auto end = fs::recursive_directory_iterator();
   for (auto it = fs::recursive_directory_iterator(workDir); it != end; ++it) {
     const auto& p = it->path();
     if (p.filename() == "Package.swift") {
-      files.push_back(p);
+      structure.packageFiles.push_back(p);
+      continue;
+    }
+    if (p.filename() == "Podfile") {
+      structure.podfiles.push_back(p);
+      continue;
+    }
+    if (p.filename() == "Cartfile" || p.filename() == "Cartfile.private") {
+      structure.cartfiles.push_back(p);
       continue;
     }
     if (!it->is_directory()) {
@@ -217,41 +233,29 @@ static std::vector<fs::path> collectFiles(const std::string& workingDir) {
       continue;
     }
     if (p.extension() == ".xcodeproj" || p.extension() == ".xcworkspace") {
-      files.push_back(p);
+      structure.xcodeFiles.push_back(p);
     }
   }
-  return files;
+  return structure;
 }
 
 static std::unordered_map<std::string, std::vector<std::string>> collectWorkspaces(
-    const std::string& workingDir,
-    bool& swiftPackageEncountered) {
+    const ProjectFiles& projectFiles) {
   // Here we are collecting list of all workspaces and Xcode projects corresponding to them
   // Projects without workspaces go into the same "empty-workspace" bucket
-  swiftPackageEncountered = false;
   std::unordered_map<std::string, std::vector<std::string>> workspaces;
   std::unordered_set<std::string> projectsBelongingToWorkspace;
-  std::vector<fs::path> files = collectFiles(workingDir);
-  for (auto& path : files) {
+  for (auto& path : projectFiles.xcodeFiles) {
     if (path.extension() == ".xcworkspace") {
       auto projects = readProjectsFromWorkspace(path.string());
       for (auto& project : projects) {
         projectsBelongingToWorkspace.insert(project.string());
         workspaces[path.string()].push_back(project.string());
       }
-    } else if (!swiftPackageEncountered && path.filename() == "Package.swift") {
-      // a package manifest must begin with a specific header comment
-      // see https://docs.swift.org/package-manager/PackageDescription/PackageDescription.html
-      static constexpr std::string_view packageHeader = "// swift-tools-version:";
-      std::array<char, packageHeader.size()> buffer{};
-      std::string_view bufferView{buffer.data(), buffer.size()};
-      if (std::ifstream{path}.read(buffer.data(), buffer.size()) && bufferView == packageHeader) {
-        swiftPackageEncountered = true;
-      }
     }
   }
   // Collect all projects not belonging to any workspace into a separate empty bucket
-  for (auto& path : files) {
+  for (auto& path : projectFiles.xcodeFiles) {
     if (path.extension() == ".xcodeproj") {
       if (projectsBelongingToWorkspace.count(path.string())) {
         continue;
@@ -262,11 +266,15 @@ static std::unordered_map<std::string, std::vector<std::string>> collectWorkspac
   return workspaces;
 }
 
-Targets collectTargets(const std::string& workingDir) {
-  Targets ret;
+ProjectStructure scanProjectStructure(const std::string& workingDir) {
+  ProjectStructure ret;
   // Getting a list of workspaces and the project that belong to them
-  auto workspaces = collectWorkspaces(workingDir, ret.swiftPackageEncountered);
+  auto projectFiles = scanWorkingDir(workingDir);
+  auto workspaces = collectWorkspaces(projectFiles);
   ret.xcodeEncountered = !workspaces.empty();
+  ret.swiftPackages = std::move(projectFiles.packageFiles);
+  ret.podfiles = std::move(projectFiles.podfiles);
+  ret.cartfiles = std::move(projectFiles.cartfiles);
   if (!ret.xcodeEncountered) {
     return ret;
   }
@@ -278,8 +286,8 @@ Targets collectTargets(const std::string& workingDir) {
   auto targetFilesMapping = mapTargetsToSourceFiles(workspaces);
 
   for (auto& [targetName, data] : targetMapping) {
-    ret.targets.push_back(Target{data.workspace, data.project, targetName, data.type,
-                                 targetFilesMapping[targetName]});
+    ret.xcodeTargets.push_back(XcodeTarget{data.workspace, data.project, targetName, data.type,
+                                           targetFilesMapping[targetName]});
   }
   return ret;
 }

swift/xcode-autobuilder/XcodeProjectParser.h

@@ -3,11 +3,17 @@
 #include "swift/xcode-autobuilder/XcodeTarget.h"
 #include <vector>
 #include <string>
+#include <filesystem>
 
-struct Targets {
-  std::vector<Target> targets;
+struct ProjectStructure {
+  std::vector<XcodeTarget> xcodeTargets;
   bool xcodeEncountered;
-  bool swiftPackageEncountered;
+  // Swift Package Manager support
+  std::vector<std::filesystem::path> swiftPackages;
+  // CocoaPods support
+  std::vector<std::filesystem::path> podfiles;
+  // Carthage support
+  std::vector<std::filesystem::path> cartfiles;
 };
 
-Targets collectTargets(const std::string& workingDir);
+ProjectStructure scanProjectStructure(const std::string& workingDir);

swift/xcode-autobuilder/XcodeTarget.h

@@ -3,12 +3,12 @@
 #include <string>
 #include <binlog/adapt_struct.hpp>
 
-struct Target {
+struct XcodeTarget {
   std::string workspace;
   std::string project;
   std::string name;
   std::string type;
   size_t fileCount;
 };
 
-BINLOG_ADAPT_STRUCT(Target, workspace, project, name, type, fileCount);
+BINLOG_ADAPT_STRUCT(XcodeTarget, workspace, project, name, type, fileCount);

swift/xcode-autobuilder/xcode-autobuilder.cpp

@@ -24,12 +24,6 @@ constexpr codeql::SwiftDiagnostic noSwiftTarget{
     .action = "To analyze a custom set of source files, set up a [manual build "
               "command][1].\n\n[1]: " MANUAL_BUILD_COMMAND_HELP_LINK};
 
-constexpr codeql::SwiftDiagnostic spmNotSupported{
-    .id = "spm-not-supported",
-    .name = "Swift Package Manager is not supported",
-    .action = "Swift Package Manager builds are not currently supported by `autobuild`. Set up a "
-              "[manual build command][1].\n\n[1]: " MANUAL_BUILD_COMMAND_HELP_LINK};
-
 static codeql::Logger& logger() {
   static codeql::Logger ret{"main"};
   return ret;
@@ -44,38 +38,63 @@ static bool endsWith(std::string_view s, std::string_view suffix) {
   return s.size() >= suffix.size() && s.substr(s.size() - suffix.size()) == suffix;
 }
 
-static bool isNonSwiftOrTestTarget(const Target& t) {
+static bool isNonSwiftOrTestTarget(const XcodeTarget& t) {
   return t.fileCount == 0 || t.type == uiTest || t.type == unitTest ||
          // unknown target types can be legitimate, let's do a name-based heuristic then
          (t.type == unknownType && (endsWith(t.name, "Tests") || endsWith(t.name, "Test")));
 }
 
-static void autobuild(const CLIArgs& args) {
-  auto collected = collectTargets(args.workingDir);
-  auto& targets = collected.targets;
-  for (const auto& t : targets) {
+static void buildSwiftPackages(const std::vector<std::filesystem::path>& swiftPackages,
+                               bool dryRun) {
+  auto any_successful =
+      std::any_of(std::begin(swiftPackages), std::end(swiftPackages), [&](auto& packageFile) {
+        LOG_INFO("Building Swift package: {}", packageFile);
+        return buildSwiftPackage(packageFile, dryRun);
+      });
+  if (!any_successful) {
+    codeql::Log::flush();
+    exit(1);
+  }
+}
+
+static bool autobuild(const CLIArgs& args) {
+  auto structure = scanProjectStructure(args.workingDir);
+  auto& xcodeTargets = structure.xcodeTargets;
+  auto& swiftPackages = structure.swiftPackages;
+  for (const auto& t : xcodeTargets) {
     LOG_INFO("{}", t);
   }
   // Filter out targets that are tests or have no swift source files
-  targets.erase(std::remove_if(std::begin(targets), std::end(targets), isNonSwiftOrTestTarget),
-                std::end(targets));
+  xcodeTargets.erase(
+      std::remove_if(std::begin(xcodeTargets), std::end(xcodeTargets), isNonSwiftOrTestTarget),
+      std::end(xcodeTargets));
 
   // Sort targets by the amount of files in each
-  std::sort(std::begin(targets), std::end(targets),
-            [](Target& lhs, Target& rhs) { return lhs.fileCount > rhs.fileCount; });
-  if ((!collected.xcodeEncountered || targets.empty()) && collected.swiftPackageEncountered) {
-    DIAGNOSE_ERROR(spmNotSupported,
-                   "A Swift package was detected, but no viable Xcode target was found.");
-  } else if (!collected.xcodeEncountered) {
-    DIAGNOSE_ERROR(noProjectFound, "`autobuild` could not detect an Xcode project or workspace.");
-  } else if (targets.empty()) {
+  std::sort(std::begin(xcodeTargets), std::end(xcodeTargets),
+            [](XcodeTarget& lhs, XcodeTarget& rhs) { return lhs.fileCount > rhs.fileCount; });
+
+  if (structure.xcodeEncountered && xcodeTargets.empty() && swiftPackages.empty()) {
+    // Report error only when there are no Xcode targets and no Swift packages
     DIAGNOSE_ERROR(noSwiftTarget, "All targets found within Xcode projects or workspaces either "
                                   "contain no Swift source files, or are tests.");
-  } else {
-    LOG_INFO("Selected {}", targets.front());
-    buildTarget(targets.front(), args.dryRun);
-    return;
+    return false;
+  } else if (!structure.xcodeEncountered && swiftPackages.empty()) {
+    DIAGNOSE_ERROR(noProjectFound,
+                   "`autobuild` could not detect an Xcode project or workspace or Swift package");
+    return false;
+  } else if (!xcodeTargets.empty()) {
+    LOG_INFO("Building Xcode target: {}", xcodeTargets.front());
+    installDependencies(structure, args.dryRun);
+    auto buildSucceeded = buildXcodeTarget(xcodeTargets.front(), args.dryRun);
+    // If build failed, try to build Swift packages
+    if (!buildSucceeded && !swiftPackages.empty()) {
+      buildSwiftPackages(swiftPackages, args.dryRun);
+    }
+    return buildSucceeded;
+  } else if (!swiftPackages.empty()) {
+    buildSwiftPackages(swiftPackages, args.dryRun);
   }
+  return true;
 }
 
 static CLIArgs parseCLIArgs(int argc, char** argv) {
@@ -96,7 +115,10 @@ static CLIArgs parseCLIArgs(int argc, char** argv) {
 
 int main(int argc, char** argv) {
   auto args = parseCLIArgs(argc, argv);
-  autobuild(args);
+  auto success = autobuild(args);
   codeql::Log::flush();
+  if (!success) {
+    return 1;
+  }
   return 0;
 }