Implement JobQueue support for pipelining

This commit adds support to Cargo's internal `JobQueue` to execute
pipelined rlib compilations actually in a pipelined fashion. This
internally invovled some refactoring to ensure we juggled the right jobs
into the right places, ensuring that as soon as an rmeta file is produce
we can start subsequent compilations but still synchronize with the
finished result of a `Build` unit.

Internally this continues to still not actually pipeline anything in the
sense that rustc doesn't support pipelining, but it's in theory all the
groundwork necessary to read a signal from rustc that a metadata file is
ready to go and then plumb that into Cargo's job scheduler.

Author: alexcrichton

src/cargo/core/compiler/build_config.rs

@@ -127,6 +127,10 @@ pub enum CompileMode {
     Test,
     /// Building a target with `rustc` (lib or bin).
     Build,
+    /// Building just the metadata of an rlib with `rustc`
+    ///
+    /// This is somewhat of a special mode, and for more information see the
+    /// documentation in `job_queue.rs`
     BuildRmeta,
     /// Building a target with `rustc` to emit `rmeta` metadata only. If
     /// `test` is true, then it is also compiled with `--test` to check it like

src/cargo/core/compiler/context/unit_dependencies.rs

@@ -660,7 +660,7 @@ fn pipeline_compilations(state: &mut State<'_, '_, '_>) -> CargoResult<()> {
         debug_assert_eq!(build_rmeta_unit.mode, CompileMode::BuildRmeta);
         debug_assert_eq!(build_unit.mode, CompileMode::Build);
 
-        // There may be multiple paths to the root of the depenency graph as we
+        // There may be multiple paths to the root of the dependency graph as we
         // walk upwards, but we don't want to add units more than once. Use a
         // visited set to guard against this.
         if !updated.insert((*build_unit, *build_rmeta_unit)) {

src/cargo/core/compiler/job_queue.rs

@@ -244,6 +244,7 @@ fn rustc<'a, 'cfg>(
         .unwrap_or_else(|| cx.bcx.config.cwd())
         .to_path_buf();
     let fingerprint_dir = cx.files().fingerprint_dir(unit);
+    let is_build_rmeta = unit.mode == CompileMode::BuildRmeta;
 
     return Ok(Work::new(move |state| {
         // Only at runtime have we discovered what the extra -L and -l
@@ -315,6 +316,25 @@ fn rustc<'a, 'cfg>(
                 .chain_err(|| format!("Could not compile `{}`.", name))?;
         }
 
+        // FIXME(rust-lang/rust#58465): this is the whole point of "pipelined
+        // compilation" in Cargo.  We want to, here in this unit, call
+        // `finish_rmeta` as soon as we can which indicates that the metadata
+        // file is emitted by rustc and ready to go. This will start dependency
+        // compilations as soon as possible.
+        //
+        // The compiler doesn't currently actually implement the ability to let
+        // us know, however, when the metadata file is ready to go. It actually
+        // today produces the file very late in compilation, far later than it
+        // would otherwise be able to do!
+        //
+        // In any case this is all covered by the issue above. This is just a
+        // marker for "yes we unconditionally do this today but tomorrow we
+        // should actually read what rustc is doing and execute this at an
+        // appropriate time, ideally long before rustc finishes completely".
+        if is_build_rmeta {
+            state.finish_rmeta();
+        }
+
         if do_rename && real_name != crate_name {
             let dst = &outputs[0].path;
             let src = dst.with_file_name(
@@ -790,11 +810,14 @@ fn build_base_args<'a, 'cfg>(
         }
     }
 
-    if unit.mode.is_check() {
-        cmd.arg("--emit=dep-info,metadata");
-    } else {
-        cmd.arg("--emit=dep-info,metadata,link");
-    }
+    cmd.arg(match unit.mode {
+        CompileMode::Build | CompileMode::Test | CompileMode::Bench => "--emit=dep-info,link",
+        CompileMode::BuildRmeta => "--emit=dep-info,metadata,link",
+        CompileMode::Check { .. } => "--emit=dep-info,metadata",
+        CompileMode::Doc { .. } | CompileMode::Doctest | CompileMode::RunCustomBuild => {
+            unreachable!()
+        }
+    });
 
     let prefer_dynamic = (unit.target.for_host() && !unit.target.is_custom_build())
         || (crate_types.contains(&"dylib") && bcx.ws.members().any(|p| p != unit.pkg));

src/cargo/core/compiler/mod.rs

@@ -124,7 +124,7 @@ impl LibKind {
     pub fn requires_upstream_objects(&self) -> bool {
         match *self {
             // "lib" == "rlib" and is a compilation that doesn't actually
-            // require upstream object files to exist, only upstream etadata
+            // require upstream object files to exist, only upstream metadata
             // files. As a result, it doesn't require upstream artifacts
             LibKind::Lib | LibKind::Rlib => false,
 

src/cargo/core/manifest.rs

@@ -132,6 +132,19 @@ impl<K: Hash + Eq + Clone, V> DependencyQueue<K, V> {
         Some((key, data))
     }
 
+    /// Forcibly dequeues a particular key from the dependency queue.
+    ///
+    /// # Panics
+    ///
+    /// Panics if `key`'s dependencies haven't finished yet or if `key` isn't in
+    /// the dependency queue.
+    pub fn dequeue_key(&mut self, key: &K) -> V {
+        let (deps, data) = self.dep_map.remove(&key).unwrap();
+        assert!(deps.is_empty());
+        self.pending.insert(key.clone());
+        return data;
+    }
+
     /// Returns `true` if there are remaining packages to be built.
     pub fn is_empty(&self) -> bool {
         self.dep_map.is_empty() && self.pending.is_empty()

src/cargo/util/dependency_queue.rs

@@ -1451,11 +1451,11 @@ fn cargo_default_env_metadata_env_var() {
 
     // No metadata on libbar since it's a dylib path dependency
     p.cargo("build -v")
-        .with_stderr(
+        .with_stderr(&format!(
             "\
 [COMPILING] bar v0.0.1 ([CWD]/bar)
 [RUNNING] `rustc --crate-name bar bar/src/lib.rs --color never --crate-type dylib \
-        --emit=[..]link \
+        --emit=dep-info,link \
         -C prefer-dynamic -C debuginfo=2 \
         -C metadata=[..] \
         --out-dir [..] \
@@ -1467,21 +1467,23 @@ fn cargo_default_env_metadata_env_var() {
         -C extra-filename=[..] \
         --out-dir [..] \
         -L dependency=[CWD]/target/debug/deps \
-        --extern bar=[CWD]/target/debug/deps/libbar[..]`
+        --extern bar=[CWD]/target/debug/deps/{prefix}bar{suffix}`
 [FINISHED] dev [unoptimized + debuginfo] target(s) in [..]",
-        )
+            prefix = env::consts::DLL_PREFIX,
+            suffix = env::consts::DLL_SUFFIX,
+        ))
         .run();
 
     p.cargo("clean").run();
 
     // If you set the env-var, then we expect metadata on libbar
     p.cargo("build -v")
         .env("__CARGO_DEFAULT_LIB_METADATA", "stable")
-        .with_stderr(
+        .with_stderr(&format!(
             "\
 [COMPILING] bar v0.0.1 ([CWD]/bar)
 [RUNNING] `rustc --crate-name bar bar/src/lib.rs --color never --crate-type dylib \
-        --emit=[..]link \
+        --emit=dep-info,link \
         -C prefer-dynamic -C debuginfo=2 \
         -C metadata=[..] \
         --out-dir [..] \
@@ -1493,10 +1495,12 @@ fn cargo_default_env_metadata_env_var() {
         -C extra-filename=[..] \
         --out-dir [..] \
         -L dependency=[CWD]/target/debug/deps \
-        --extern bar=[CWD]/target/debug/deps/libbar-[..]`
+        --extern bar=[CWD]/target/debug/deps/{prefix}bar-[..]{suffix}`
 [FINISHED] dev [unoptimized + debuginfo] target(s) in [..]
 ",
-        )
+            prefix = env::consts::DLL_PREFIX,
+            suffix = env::consts::DLL_SUFFIX,
+        ))
         .run();
 }
 
@@ -1921,9 +1925,6 @@ fn verbose_release_build_deps() {
             version = "0.0.0"
             authors = []
 
-            [lib]
-            crate-type = ['cdylib']
-
             [dependencies.foo]
             path = "foo"
         "#,
@@ -1958,7 +1959,7 @@ fn verbose_release_build_deps() {
         --out-dir [..] \
         -L dependency=[CWD]/target/release/deps`
 [COMPILING] test v0.0.0 ([CWD])
-[RUNNING] `rustc --crate-name test src/lib.rs --color never --crate-type cdylib \
+[RUNNING] `rustc --crate-name test src/lib.rs --color never --crate-type lib \
         --emit=[..]link \
         -C opt-level=3 \
         -C metadata=[..] \

tests/testsuite/build.rs

@@ -61,7 +61,6 @@ fn build_dep_info_rlib() {
         .build();
 
     p.cargo("build --example=ex")
-        .stream()
         .env("RUST_LOG", "cargo")
         .run();
     assert!(p.example_lib("ex", "rlib").with_extension("d").is_file());