Add can_fail flag to Unit that allows compilation to proceed even if

specific units fail. All Docscrape units now have can_fail = true to
avoid stopping Rustdoc if an example fails to scrape.

Author: willcrichton

src/cargo/core/compiler/context/mod.rs

@@ -77,6 +77,13 @@ pub struct Context<'a, 'cfg> {
     /// Map of Doc/Docscrape units to metadata for their -Cmetadata flag.
     /// See Context::find_metadata_units for more details.
     pub metadata_for_doc_units: HashMap<Unit, Metadata>,
+
+    /// Map that tracks whether a unit completed successfully. Used in conjuction
+    /// with the `Unit::can_fail` flag, so jobs can dynamically track at runtime
+    /// whether their dependencies succeeded or failed. Currently used to for
+    /// the Rustdoc scrape-examples feature to allow Rustdoc to proceed even if
+    /// examples fail to scrape.
+    pub completed_units: Arc<Mutex<HashMap<Metadata, bool>>>,
 }
 
 impl<'a, 'cfg> Context<'a, 'cfg> {
@@ -115,6 +122,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
             rustc_clients: HashMap::new(),
             lto: HashMap::new(),
             metadata_for_doc_units: HashMap::new(),
+            completed_units: Arc::new(Mutex::new(HashMap::new())),
         })
     }
 

src/cargo/core/compiler/job_queue.rs

@@ -126,6 +126,7 @@ struct DrainState<'cfg> {
     total_units: usize,
 
     queue: DependencyQueue<Unit, Artifact, Job>,
+    dep_map: HashMap<Unit, HashSet<(Unit, Artifact)>>,
     messages: Arc<Queue<Message>>,
     /// Diagnostic deduplication support.
     diag_dedupe: DiagDedupe<'cfg>,
@@ -508,6 +509,12 @@ impl<'cfg> JobQueue<'cfg> {
         let progress = Progress::with_style("Building", ProgressStyle::Ratio, cx.bcx.config);
         let state = DrainState {
             total_units: self.queue.len(),
+            dep_map: self
+                .queue
+                .dep_map()
+                .iter()
+                .map(|(unit, (deps, _))| (unit.clone(), deps.clone()))
+                .collect(),
             queue: self.queue,
             // 100 here is somewhat arbitrary. It is a few screenfulls of
             // output, and hopefully at most a few megabytes of memory for
@@ -578,6 +585,32 @@ impl<'cfg> DrainState<'cfg> {
         // start requesting job tokens. Each job after the first needs to
         // request a token.
         while let Some((unit, job)) = self.queue.dequeue() {
+            // First, we handle the special case of fallible units. If
+            // this unit is allowed to fail, and any one of its dependencies
+            // has failed, then we should immediately mark it as failed and
+            // skip executing it.
+            if unit.can_fail {
+                let mut completed_units = cx.completed_units.lock().unwrap();
+                let failed_deps = self.dep_map[&unit]
+                    .iter()
+                    .filter(|(dep_unit, _)| {
+                        let dep_meta = cx.files().metadata(dep_unit);
+                        !completed_units[&dep_meta]
+                    })
+                    .map(|(_, artifact)| artifact)
+                    .collect::<HashSet<_>>();
+                if !failed_deps.is_empty() {
+                    // TODO: should put a warning here saying which units were skipped
+                    // due to failed dependencies.
+                    for artifact in failed_deps {
+                        self.queue.finish(&unit, artifact);
+                    }
+                    let unit_meta = cx.files().metadata(&unit);
+                    completed_units.insert(unit_meta, false);
+                    continue;
+                }
+            }
+
             self.pending_queue.push((unit, job));
             if self.active.len() + self.pending_queue.len() > 1 {
                 jobserver_helper.request_token();
@@ -713,7 +746,8 @@ impl<'cfg> DrainState<'cfg> {
                 };
                 debug!("end ({:?}): {:?}", unit, result);
                 match result {
-                    Ok(()) => self.finish(id, &unit, artifact, cx)?,
+                    Ok(()) => self.finish(id, &unit, artifact, cx, true)?,
+                    Err(_) if unit.can_fail => self.finish(id, &unit, artifact, cx, false)?,
                     Err(error) => {
                         let msg = "The following warnings were emitted during compilation:";
                         self.emit_warnings(Some(msg), &unit, cx)?;
@@ -1161,6 +1195,7 @@ impl<'cfg> DrainState<'cfg> {
         unit: &Unit,
         artifact: Artifact,
         cx: &mut Context<'_, '_>,
+        success: bool,
     ) -> CargoResult<()> {
         if unit.mode.is_run_custom_build() && unit.show_warnings(cx.bcx.config) {
             self.emit_warnings(None, unit, cx)?;
@@ -1170,6 +1205,11 @@ impl<'cfg> DrainState<'cfg> {
             Artifact::All => self.timings.unit_finished(id, unlocked),
             Artifact::Metadata => self.timings.unit_rmeta_finished(id, unlocked),
         }
+        cx.completed_units
+            .lock()
+            .unwrap()
+            .insert(cx.files().metadata(unit), success);
+
         Ok(())
     }
 

src/cargo/core/compiler/mod.rs

@@ -22,13 +22,13 @@ mod unit;
 pub mod unit_dependencies;
 pub mod unit_graph;
 
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 use std::env;
 use std::ffi::{OsStr, OsString};
 use std::fs::{self, File};
 use std::io::{BufRead, Write};
 use std::path::{Path, PathBuf};
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 
 use anyhow::{Context as _, Error};
 use lazycell::LazyCell;
@@ -263,6 +263,7 @@ fn rustc(cx: &mut Context<'_, '_>, unit: &Unit, exec: &Arc<dyn Executor>) -> Car
     let script_metadata = cx.find_build_script_metadata(unit);
     let is_local = unit.is_local();
     let artifact = unit.artifact;
+    let completed_units = Arc::clone(&cx.completed_units);
 
     return Ok(Work::new(move |state| {
         // Artifacts are in a different location than typical units,
@@ -292,7 +293,13 @@ fn rustc(cx: &mut Context<'_, '_>, unit: &Unit, exec: &Arc<dyn Executor>) -> Car
                 )?;
                 add_plugin_deps(&mut rustc, &script_outputs, &build_scripts, &root_output)?;
             }
-            add_custom_flags(&mut rustc, &script_outputs, script_metadata)?;
+            add_custom_flags(
+                &mut rustc,
+                &script_outputs,
+                script_metadata,
+                completed_units,
+                None,
+            )?;
         }
 
         for output in outputs.iter() {
@@ -639,9 +646,9 @@ fn rustdoc(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Work> {
     let metadata = cx.metadata_for_doc_units[unit];
     rustdoc.arg("-C").arg(format!("metadata={}", metadata));
 
-    let scrape_output_path = |unit: &Unit| -> CargoResult<PathBuf> {
+    let scrape_output_path = |unit: &Unit| -> PathBuf {
         let output_dir = cx.files().deps_dir(unit);
-        Ok(output_dir.join(format!("{}.examples", unit.buildkey())))
+        output_dir.join(format!("{}.examples", unit.buildkey()))
     };
 
     if unit.mode.is_doc_scrape() {
@@ -651,7 +658,7 @@ fn rustdoc(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Work> {
 
         rustdoc
             .arg("--scrape-examples-output-path")
-            .arg(scrape_output_path(unit)?);
+            .arg(scrape_output_path(unit));
 
         // Only scrape example for items from crates in the workspace, to reduce generated file size
         for pkg in cx.bcx.ws.members() {
@@ -664,18 +671,24 @@ fn rustdoc(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Work> {
                 rustdoc.arg("--scrape-examples-target-crate").arg(name);
             }
         }
-    } else if cx.bcx.scrape_units.len() > 0 && cx.bcx.ws.unit_needs_doc_scrape(unit) {
+    }
+
+    let scrape_units = if cx.bcx.scrape_units.len() > 0 && cx.bcx.ws.unit_needs_doc_scrape(unit) {
         // We only pass scraped examples to packages in the workspace
         // since examples are only coming from reverse-dependencies of workspace packages
 
         rustdoc.arg("-Zunstable-options");
 
-        for scrape_unit in &cx.bcx.scrape_units {
-            rustdoc
-                .arg("--with-examples")
-                .arg(scrape_output_path(scrape_unit)?);
-        }
-    }
+        Some(
+            cx.bcx
+                .scrape_units
+                .iter()
+                .map(|unit| (cx.files().metadata(unit), scrape_output_path(unit)))
+                .collect::<HashMap<_, _>>(),
+        )
+    } else {
+        None
+    };
 
     build_deps_args(&mut rustdoc, cx, unit)?;
     rustdoc::add_root_urls(cx, unit, &mut rustdoc)?;
@@ -693,11 +706,14 @@ fn rustdoc(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Work> {
     let target = Target::clone(&unit.target);
     let mut output_options = OutputOptions::new(cx, unit);
     let script_metadata = cx.find_build_script_metadata(unit);
+    let completed_units = Arc::clone(&cx.completed_units);
     Ok(Work::new(move |state| {
         add_custom_flags(
             &mut rustdoc,
             &build_script_outputs.lock().unwrap(),
             script_metadata,
+            completed_units,
+            scrape_units,
         )?;
         let crate_dir = doc_dir.join(&crate_name);
         if crate_dir.exists() {
@@ -706,6 +722,7 @@ fn rustdoc(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Work> {
             debug!("removing pre-existing doc directory {:?}", crate_dir);
             paths::remove_dir_all(crate_dir)?;
         }
+
         state.running(&rustdoc);
 
         rustdoc
@@ -1165,6 +1182,8 @@ fn add_custom_flags(
     cmd: &mut ProcessBuilder,
     build_script_outputs: &BuildScriptOutputs,
     metadata: Option<Metadata>,
+    completed_units: Arc<Mutex<HashMap<Metadata, bool>>>,
+    scrape_units: Option<HashMap<Metadata, PathBuf>>,
 ) -> CargoResult<()> {
     if let Some(metadata) = metadata {
         if let Some(output) = build_script_outputs.get(metadata) {
@@ -1183,6 +1202,16 @@ fn add_custom_flags(
         }
     }
 
+    if let Some(scrape_units) = scrape_units {
+        let completed_units = completed_units.lock().unwrap();
+        for (_, output_path) in scrape_units
+            .iter()
+            .filter(|(buildkey, _)| completed_units[*buildkey])
+        {
+            cmd.arg("--with-examples").arg(output_path);
+        }
+    }
+
     Ok(())
 }
 

src/cargo/core/compiler/standard_lib.rs

@@ -217,6 +217,7 @@ pub fn generate_std_roots(
                 /*is_std*/ true,
                 /*dep_hash*/ 0,
                 IsArtifact::No,
+                false,
             ));
         }
     }

src/cargo/core/compiler/unit.rs

@@ -72,6 +72,7 @@ pub struct UnitInner {
     /// This value initially starts as 0, and then is filled in via a
     /// second-pass after all the unit dependencies have been computed.
     pub dep_hash: u64,
+    pub can_fail: bool,
 }
 
 impl UnitInner {
@@ -141,6 +142,7 @@ impl fmt::Debug for Unit {
             .field("artifact", &self.artifact.is_true())
             .field("is_std", &self.is_std)
             .field("dep_hash", &self.dep_hash)
+            .field("can_fail", &self.can_fail)
             .finish()
     }
 }
@@ -184,6 +186,7 @@ impl UnitInterner {
         is_std: bool,
         dep_hash: u64,
         artifact: IsArtifact,
+        can_fail: bool,
     ) -> Unit {
         let target = match (is_std, target.kind()) {
             // This is a horrible hack to support build-std. `libstd` declares
@@ -216,6 +219,7 @@ impl UnitInterner {
             is_std,
             dep_hash,
             artifact,
+            can_fail,
         });
         Unit { inner }
     }

src/cargo/core/compiler/unit_dependencies.rs

@@ -888,6 +888,7 @@ fn new_unit_dep_with_profile(
         state.is_std,
         /*dep_hash*/ 0,
         artifact.map_or(IsArtifact::No, |_| IsArtifact::Yes),
+        parent.can_fail,
     );
     Ok(UnitDep {
         unit,

src/cargo/core/workspace.rs

@@ -1517,7 +1517,7 @@ impl<'cfg> Workspace<'cfg> {
         // (not documented) or proc macros (have no scrape-able exports). Additionally,
         // naively passing a proc macro's unit_for to new_unit_dep will currently cause
         // Cargo to panic, see issue #10545.
-        self.is_member(&unit.pkg) && !unit.target.for_host()
+        self.is_member(&unit.pkg) && !unit.target.for_host() && unit.mode.is_doc()
     }
 }
 

src/cargo/ops/cargo_compile.rs

@@ -1069,6 +1069,7 @@ fn generate_targets(
                 /*is_std*/ false,
                 /*dep_hash*/ 0,
                 IsArtifact::No,
+                mode.is_doc_scrape(),
             );
             units.insert(unit);
         }
@@ -1631,6 +1632,7 @@ fn traverse_and_share(
         unit.is_std,
         new_dep_hash,
         unit.artifact,
+        unit.can_fail,
     );
     assert!(memo.insert(unit.clone(), new_unit.clone()).is_none());
     new_graph.entry(new_unit.clone()).or_insert(new_deps);
@@ -1872,6 +1874,7 @@ fn override_rustc_crate_types(
             unit.is_std,
             unit.dep_hash,
             unit.artifact,
+            unit.can_fail,
         )
     };
     units[0] = match unit.target.kind() {

src/cargo/util/dependency_queue.rs

@@ -170,6 +170,10 @@ impl<N: Hash + Eq + Clone, E: Eq + Hash + Clone, V> DependencyQueue<N, E, V> {
         self.dep_map.len()
     }
 
+    pub fn dep_map(&self) -> &HashMap<N, (HashSet<(N, E)>, V)> {
+        &self.dep_map
+    }
+
     /// Indicate that something has finished.
     ///
     /// Calling this function indicates that the `node` has produced `edge`. All