Handle transitive All dependency edges

We need to synthesize some dependency edges in the dependency graph to
get everything ordered correctly! (more details in the commit itself)

Author: alexcrichton

src/cargo/core/compiler/fingerprint.rs

@@ -313,17 +313,24 @@ pub fn prepare_target<'a, 'cfg>(
     Ok(Job::new(write_fingerprint, Dirty))
 }
 
-/// A compilation unit dependency has a fingerprint that is comprised of:
-/// * its package ID
-/// * its extern crate name
-/// * its public/private status
-/// * its calculated fingerprint for the dependency
+/// Dependency edge information for fingerprints. This is generated for each
+/// unit in `dep_targets` and is stored in a `Fingerprint` below.
 #[derive(Clone)]
 struct DepFingerprint {
+    /// The hash of the package id that this dependency points to
     pkg_id: u64,
+    /// The crate name we're using for this dependency, which if we change we'll
+    /// need to recompile!
     name: String,
+    /// Whether or not this dependency is flagged as a public dependency or not.
     public: bool,
+    /// Whether or not this dependency is an rmeta dependency or a "full"
+    /// dependency. In the case of an rmeta dependency our dependency edge only
+    /// actually requires the rmeta from what we depend on, so when checking
+    /// mtime information all files other than the rmeta can be ignored.
     only_requires_rmeta: bool,
+    /// The dependency's fingerprint we recursively point to, containing all the
+    /// other hash information we'd otherwise need.
     fingerprint: Arc<Fingerprint>,
 }
 
@@ -442,11 +449,14 @@ impl<'de> Deserialize<'de> for DepFingerprint {
             pkg_id,
             name,
             public,
-            only_requires_rmeta: false,
             fingerprint: Arc::new(Fingerprint {
                 memoized_hash: Mutex::new(Some(hash)),
                 ..Fingerprint::new()
             }),
+            // This field is never read since it's only used in
+            // `check_filesystem` which isn't used by fingerprints loaded from
+            // disk.
+            only_requires_rmeta: false,
         })
     }
 }
@@ -878,7 +888,7 @@ impl hash::Hash for Fingerprint {
             name,
             public,
             fingerprint,
-            only_requires_rmeta: _,
+            only_requires_rmeta: _, // static property, no need to hash
         } in deps
         {
             pkg_id.hash(h);

src/cargo/core/compiler/job_queue.rs

@@ -1,5 +1,5 @@
-use std::collections::{HashMap, HashSet};
 use std::cell::Cell;
+use std::collections::{HashMap, HashSet};
 use std::io;
 use std::marker;
 use std::sync::mpsc::{channel, Receiver, Sender};
@@ -152,7 +152,7 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> {
         job: Job,
     ) -> CargoResult<()> {
         let dependencies = cx.dep_targets(unit);
-        let dependencies = dependencies
+        let mut queue_deps = dependencies
             .iter()
             .filter(|unit| {
                 // Binaries aren't actually needed to *compile* tests, just to run
@@ -171,10 +171,52 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> {
                     Artifact::All
                 };
                 (dep, artifact)
-            });
-        self.queue.queue(*unit, job, dependencies);
+            })
+            .collect::<Vec<_>>();
+
+        // This is somewhat tricky, but we may need to synthesize some
+        // dependencies for this target if it requires full upstream
+        // compilations to have completed. If we're in pipelining mode then some
+        // dependency edges may be `Metadata` due to the above clause (as
+        // opposed to everything being `All`). For example consider:
+        //
+        //    a (binary)
+        //    └ b (lib)
+        //        └ c (lib)
+        //
+        // Here the dependency edge from B to C will be `Metadata`, and the
+        // dependency edge from A to B will be `All`. For A to be compiled,
+        // however, it currently actually needs the full rlib of C. This means
+        // that we need to synthesize a dependency edge for the dependency graph
+        // from A to C. That's done here.
+        //
+        // This will walk all dependencies of the current target, and if any of
+        // *their* dependencies are `Metadata` then we depend on the `All` of
+        // the target as well. This should ensure that edges changed to
+        // `Metadata` propagate upwards `All` dependencies to anything that
+        // transitively contains the `Metadata` edge.
+        if unit.target.requires_upstream_objects() {
+            for dep in dependencies.iter() {
+                depend_on_deps_of_deps(cx, &mut queue_deps, dep);
+            }
+
+            fn depend_on_deps_of_deps<'a>(
+                cx: &Context<'a, '_>,
+                deps: &mut Vec<(Unit<'a>, Artifact)>,
+                unit: &Unit<'a>,
+            ) {
+                for dep in cx.dep_targets(unit) {
+                    if cx.only_requires_rmeta(unit, &dep) {
+                        deps.push((dep, Artifact::All));
+                        depend_on_deps_of_deps(cx, deps, &dep);
+                    }
+                }
+            }
+        }
+
+        self.queue.queue(*unit, job, queue_deps);
         *self.counts.entry(unit.pkg.package_id()).or_insert(0) += 1;
-        Ok(())
+        return Ok(());
     }
 
     /// Executes all jobs necessary to build the dependency graph.
@@ -430,7 +472,6 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> {
         cx: &Context<'a, '_>,
         scope: &Scope<'a>,
     ) -> CargoResult<()> {
-
         let id = self.next_id;
         self.next_id = id.checked_add(1).unwrap();
 
@@ -464,7 +505,9 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> {
             // we need to make sure that the metadata is flagged as produced so
             // send a synthetic message here.
             if state.rmeta_required.get() && res.is_ok() {
-                my_tx.send(Message::Finish(id, Artifact::Metadata, Ok(()))).unwrap();
+                my_tx
+                    .send(Message::Finish(id, Artifact::Metadata, Ok(())))
+                    .unwrap();
             }
 
             my_tx.send(Message::Finish(id, Artifact::All, res)).unwrap();
@@ -513,7 +556,12 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> {
         Ok(())
     }
 
-    fn finish(&mut self, unit: &Unit<'a>, artifact: Artifact, cx: &mut Context<'_, '_>) -> CargoResult<()> {
+    fn finish(
+        &mut self,
+        unit: &Unit<'a>,
+        artifact: Artifact,
+        cx: &mut Context<'_, '_>,
+    ) -> CargoResult<()> {
         if unit.mode.is_run_custom_build() && cx.bcx.show_warnings(unit.pkg.package_id()) {
             self.emit_warnings(None, unit, cx)?;
         }