Merge pull request swiftlang#35036 from CodaFi/acrosstic-poetry

Move The Last Pieces for Cross-Module Incremental Builds

Author: CodaFi

include/swift/Basic/FileTypes.def

@@ -61,6 +61,7 @@ TYPE("llvm-bc",             LLVM_BC,                   "bc",              "")
 TYPE("diagnostics",         SerializedDiagnostics,     "dia",             "")
 TYPE("objc-header",         ObjCHeader,                "h",               "")
 TYPE("swift-dependencies",  SwiftDeps,                 "swiftdeps",       "")
+TYPE("external-swift-dependencies", ExternalSwiftDeps,  "swiftdeps.external",  "")
 TYPE("swift-ranges",        SwiftRanges,               "swiftranges",     "")
 TYPE("compiled-source",     CompiledSource,            "compiledsource",  "")
 TYPE("remap",               Remapping,                 "remap",           "")

include/swift/Basic/STLExtras.h

@@ -27,6 +27,7 @@
 #include <iterator>
 #include <numeric>
 #include <type_traits>
+#include <unordered_set>
 
 namespace swift {
 
@@ -752,6 +753,22 @@ using all_true =
 template <class... Ts>
 using are_all_compound = all_true<std::is_compound<Ts>::value...>;
 
+/// Erase all elements in \p c that match the given predicate \p pred.
+// FIXME: Remove this when C++20 is the new baseline.
+template <class Key, class Hash, class KeyEqual, class Alloc, class Pred>
+typename std::unordered_set<Key, Hash, KeyEqual, Alloc>::size_type
+erase_if(std::unordered_set<Key, Hash, KeyEqual, Alloc> &c, Pred pred) {
+  auto startingSize = c.size();
+  for (auto i = c.begin(), last = c.end(); i != last;) {
+    if (pred(*i)) {
+      i = c.erase(i);
+    } else {
+      ++i;
+    }
+  }
+  return startingSize - c.size();
+}
+
 } // end namespace swift
 
 #endif // SWIFT_BASIC_INTERLEAVE_H

include/swift/Driver/Compilation.h

@@ -508,6 +508,17 @@ class Compilation {
   /// \sa types::isPartOfSwiftCompilation
   const char *getAllSourcesPath() const;
 
+  /// Retrieve the path to the external swift deps file.
+  ///
+  /// For cross-module incremental builds, this file contains the dependencies
+  /// from all the modules integrated over the prior build.
+  ///
+  /// Currently this patch is relative to the build record, but we may want
+  /// to allow the output file map to customize this at some point.
+  std::string getExternalSwiftDepsFilePath() const {
+    return CompilationRecordPath + ".external";
+  }
+
   /// Asks the Compilation to perform the Jobs which it knows about.
   ///
   /// \param TQ The TaskQueue used to schedule jobs for execution.

lib/AST/FrontendSourceFileDepGraphFactory.cpp

@@ -538,6 +538,4 @@ void ModuleDepGraphFactory::addAllDefinedDecls() {
       declFinder.potentialMemberHolders);
   addAllDefinedDeclsOfAGivenType<NodeKind::member>(
       declFinder.valuesInExtensions);
-  addAllDefinedDeclsOfAGivenType<NodeKind::dynamicLookup>(
-      declFinder.classMembers);
 }

lib/Basic/FileTypes.cpp

@@ -100,6 +100,7 @@ bool file_types::isTextual(ID Id) {
   case file_types::TY_SerializedDiagnostics:
   case file_types::TY_ClangModuleFile:
   case file_types::TY_SwiftDeps:
+  case file_types::TY_ExternalSwiftDeps:
   case file_types::TY_SwiftRanges:
   case file_types::TY_CompiledSource:
   case file_types::TY_Nothing:
@@ -145,6 +146,7 @@ bool file_types::isAfterLLVM(ID Id) {
   case file_types::TY_SerializedDiagnostics:
   case file_types::TY_ClangModuleFile:
   case file_types::TY_SwiftDeps:
+  case file_types::TY_ExternalSwiftDeps:
   case file_types::TY_SwiftRanges:
   case file_types::TY_CompiledSource:
   case file_types::TY_Nothing:
@@ -197,6 +199,7 @@ bool file_types::isPartOfSwiftCompilation(ID Id) {
   case file_types::TY_SerializedDiagnostics:
   case file_types::TY_ClangModuleFile:
   case file_types::TY_SwiftDeps:
+  case file_types::TY_ExternalSwiftDeps:
   case file_types::TY_SwiftRanges:
   case file_types::TY_CompiledSource:
   case file_types::TY_Nothing:

lib/Driver/Compilation.cpp

@@ -15,6 +15,7 @@
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/AST/DiagnosticsDriver.h"
 #include "swift/AST/FineGrainedDependencies.h"
+#include "swift/AST/FineGrainedDependencyFormat.h"
 #include "swift/Basic/OutputFileMap.h"
 #include "swift/Basic/Program.h"
 #include "swift/Basic/STLExtras.h"
@@ -1185,6 +1186,30 @@ namespace driver {
       return ExternallyDependentJobs;
     }
 
+    using ChangeSet = fine_grained_dependencies::ModuleDepGraph::Changes::value_type;
+    static void
+    pruneChangeSetFromExternalDependency(ChangeSet &changes) {
+      // The changeset includes detritus from the graph that gets consed up
+      // in \c writePriorDependencyGraph. We need to ignore the fake
+      // source file provides nodes and the fake incremental external
+      // dependencies linked to them.
+      swift::erase_if(
+          changes, [&](fine_grained_dependencies::ModuleDepGraphNode *node) {
+            if (node->getKey().getKind() ==
+                    fine_grained_dependencies::NodeKind::sourceFileProvide ||
+                node->getKey().getKind() ==
+                    fine_grained_dependencies::NodeKind::
+                        incrementalExternalDepend) {
+              return true;
+            }
+            if (node->getKey().getAspect() ==
+                fine_grained_dependencies::DeclAspect::implementation) {
+              return true;
+            }
+            return !node->getIsProvides();
+          });
+    }
+  
     SmallVector<const Job *, 16>
     collectIncrementalExternallyDependentJobsFromDependencyGraph(
         const bool forRanges) {
@@ -1196,6 +1221,17 @@ namespace driver {
         }
       };
 
+      // Load our priors, which are always adjacent to the build record. We
+      // don't care if this load succeeds or not. If it fails, and we succeed at
+      // integrating one of the external files below, the changeset will be the
+      // entire module!
+      const auto *externalPriorJob = Comp.addExternalJob(
+          std::make_unique<Job>(Comp.getDerivedOutputFileMap(),
+                                Comp.getExternalSwiftDepsFilePath()));
+      getFineGrainedDepGraph(forRanges).loadFromPath(
+          externalPriorJob, Comp.getExternalSwiftDepsFilePath(),
+          Comp.getDiags());
+
       for (auto external : getFineGrainedDepGraph(forRanges)
                                .getIncrementalExternalDependencies()) {
         llvm::sys::fs::file_status depStatus;
@@ -1231,20 +1267,23 @@ namespace driver {
         // code's internal invariants.
         const auto *externalJob = Comp.addExternalJob(
             std::make_unique<Job>(Comp.getDerivedOutputFileMap(), external));
-        auto subChanges =
+        auto maybeChanges =
             getFineGrainedDepGraph(forRanges).loadFromSwiftModuleBuffer(
                 externalJob, *buffer.get(), Comp.getDiags());
 
         // If the incremental dependency graph failed to load, fall back to
         // treating this as plain external job.
-        if (!subChanges.hasValue()) {
+        if (!maybeChanges.hasValue()) {
           fallbackToExternalBehavior(external);
           continue;
         }
 
-        for (auto *CMD :
-             getFineGrainedDepGraph(forRanges)
-                 .findJobsToRecompileWhenNodesChange(subChanges.getValue())) {
+        // Prune away the detritus from the build record.
+        auto &changes = maybeChanges.getValue();
+        pruneChangeSetFromExternalDependency(changes);
+
+        for (auto *CMD : getFineGrainedDepGraph(forRanges)
+                             .findJobsToRecompileWhenNodesChange(changes)) {
           if (CMD == externalJob) {
             continue;
           }
@@ -1861,6 +1900,76 @@ static void writeCompilationRecord(StringRef path, StringRef argsHash,
   }
 }
 
+using SourceFileDepGraph = swift::fine_grained_dependencies::SourceFileDepGraph;
+
+/// Render out the unified module dependency graph to the given \p path, which
+/// is expected to be a path relative to the build record.
+static void withPriorDependencyGraph(StringRef path,
+                                     const Compilation::Result &result,
+                                     llvm::function_ref<void(SourceFileDepGraph &&)> cont) {
+  // Building a source file dependency graph from the module dependency graph
+  // is a strange task on its face because a source file dependency graph is
+  // usually built for exactly one file. However, the driver is going to use
+  // some encoding tricks to get the dependencies for each incremental external
+  // dependency into one big file. Note that these tricks
+  // are undone in \c pruneChangeSetFromExternalDependency, so if you modify
+  // this you need to go fix that algorithm up as well. This is a diagrammatic
+  // view of the structure of the dependencies this function builds:
+  //
+  // SourceFile => interface <BUILD_RECORD>.external
+  // | - Incremetal External Dependency => interface <MODULE_1>.swiftmodule
+  // | | - <dependency> ...
+  // | | - <dependency> ...
+  // | | - <dependency> ...
+  // | - Incremetal External Dependency => interface <MODULE_2>.swiftmodule
+  // | | - <dependency> ...
+  // | | - <dependency> ...
+  // | - Incremetal External Dependency => interface <MODULE_3>.swiftmodule
+  // | - ...
+  //
+  // Where each <dependency> node has an arc back to its parent swiftmodule.
+  // That swiftmodule, in turn, takes the form of as an incremental external
+  // dependency. This formulation allows us to easily discern the original
+  // swiftmodule that a <dependency> came from just by examining that arc. This
+  // is used in integrate to "move" the <dependency> from the build record to
+  // the swiftmodule by swapping the key it uses.
+  using namespace swift::fine_grained_dependencies;
+  SourceFileDepGraph g;
+  const auto &resultModuleGraph = result.depGraph;
+  // Create the key for the entire external build record.
+  auto fileKey =
+      DependencyKey::createKeyForWholeSourceFile(DeclAspect::interface, path);
+  auto fileNodePair = g.findExistingNodePairOrCreateAndAddIfNew(fileKey, None);
+  for (StringRef incrExternalDep :
+       resultModuleGraph.getIncrementalExternalDependencies()) {
+    // Now make a node for each incremental external dependency.
+    auto interfaceKey =
+        DependencyKey(NodeKind::incrementalExternalDepend,
+                      DeclAspect::interface, "", incrExternalDep.str());
+    auto ifaceNode = g.findExistingNodeOrCreateIfNew(interfaceKey, None,
+                                                     false /* = !isProvides */);
+    resultModuleGraph.forEachNodeInJob(incrExternalDep, [&](const auto *node) {
+      // Reject
+      // 1) Implementation nodes: We don't care about the interface nodes
+      //    for cross-module dependencies because the client cannot observe it
+      //    by definition.
+      // 2) Source file nodes: we're about to define our own.
+      if (!node->getKey().isInterface() ||
+          node->getKey().getKind() == NodeKind::sourceFileProvide) {
+        return;
+      }
+      assert(node->getIsProvides() &&
+             "Found a node in module depdendencies that is not a provides!");
+      auto *newNode = new SourceFileDepGraphNode(
+          node->getKey(), node->getFingerprint(), /*isProvides*/ true);
+      g.addNode(newNode);
+      g.addArc(ifaceNode, newNode);
+    });
+    g.addArc(fileNodePair.getInterface(), ifaceNode);
+  }
+  return cont(std::move(g));
+}
+
 static void writeInputJobsToFilelist(llvm::raw_fd_ostream &out, const Job *job,
                                      const file_types::ID infoType) {
   // FIXME: Duplicated from ToolChains.cpp.
@@ -1960,6 +2069,15 @@ Compilation::performJobsImpl(std::unique_ptr<TaskQueue> &&TQ) {
     auto result = std::move(State).takeResult();
     writeCompilationRecord(CompilationRecordPath, ArgsHash, BuildStartTime,
                            InputInfo);
+    if (EnableCrossModuleIncrementalBuild) {
+      // Write out our priors adjacent to the build record so we can pick
+      // the up in a subsequent build.
+      withPriorDependencyGraph(getExternalSwiftDepsFilePath(), result,
+                               [&](SourceFileDepGraph &&g) {
+        writeFineGrainedDependencyGraphToPath(
+            Diags, getExternalSwiftDepsFilePath(), g);
+      });
+    }
     return result;
   } else {
     return std::move(State).takeResult();

lib/Driver/Driver.cpp

@@ -2039,6 +2039,7 @@ void Driver::buildActions(SmallVectorImpl<const Action *> &TopLevelActions,
       case file_types::TY_ObjCHeader:
       case file_types::TY_ClangModuleFile:
       case file_types::TY_SwiftDeps:
+      case file_types::TY_ExternalSwiftDeps:
       case file_types::TY_SwiftRanges:
       case file_types::TY_CompiledSource:
       case file_types::TY_Remapping:

lib/Driver/FineGrainedDependencyDriverGraph.cpp

@@ -287,15 +287,29 @@ ModuleDepGraph::Changes ModuleDepGraph::integrate(const SourceFileDepGraph &g,
 
   g.forEachNode([&](const SourceFileDepGraphNode *integrand) {
     const auto &key = integrand->getKey();
-
-    auto preexistingMatch = findPreexistingMatch(swiftDepsOfJob, integrand);
+    StringRef realSwiftDepsPath = swiftDepsOfJob;
+    // If we're doing a cross-module incremental build, we'll see these
+    // `.external` "swiftdeps" files. See \c writePriorDependencyGraph for
+    // the structure of the graph we're traversing here. Essentially, follow
+    // the arc laid down there to discover the file path for the swiftmodule
+    // where this dependency node originally came from.
+    if (swiftDepsOfJob.endswith(file_types::getExtension(file_types::TY_ExternalSwiftDeps)) &&
+        integrand->getKey().getKind() != NodeKind::sourceFileProvide) {
+      integrand->forEachDefIDependUpon([&](size_t seqNum) {
+        auto &external = g.getNode(seqNum)->getKey();
+        if (external.getKind() == NodeKind::incrementalExternalDepend) {
+          realSwiftDepsPath = external.getName();
+        }
+      });
+    }
+    auto preexistingMatch = findPreexistingMatch(realSwiftDepsPath, integrand);
     if (preexistingMatch.hasValue() &&
         preexistingMatch.getValue().first == LocationOfPreexistingNode::here)
       disappearedNodes.erase(key); // Node was and still is. Do not erase it.
 
     Optional<NullablePtr<ModuleDepGraphNode>> newNodeOrChangedNode =
         integrateSourceFileDepGraphNode(g, integrand, preexistingMatch,
-                                        swiftDepsOfJob);
+                                        realSwiftDepsPath);
 
     if (!newNodeOrChangedNode)
       changedNodes = None;

lib/Driver/ToolChains.cpp

@@ -619,6 +619,7 @@ const char *ToolChain::JobContext::computeFrontendModeForCompile() const {
   case file_types::TY_ObjCHeader:
   case file_types::TY_Image:
   case file_types::TY_SwiftDeps:
+  case file_types::TY_ExternalSwiftDeps:
   case file_types::TY_SwiftRanges:
   case file_types::TY_CompiledSource:
   case file_types::TY_ModuleTrace:
@@ -881,6 +882,7 @@ ToolChain::constructInvocation(const BackendJobAction &job,
     case file_types::TY_ObjCHeader:
     case file_types::TY_Image:
     case file_types::TY_SwiftDeps:
+    case file_types::TY_ExternalSwiftDeps:
     case file_types::TY_SwiftRanges:
     case file_types::TY_CompiledSource:
     case file_types::TY_Remapping: