Add support for -Zbuild-std

- Update to a version of rustwide that supports `fetch_build_std_targets`
- Don't try to download rust-std for tier 3 targets
- Add a local test
- Don't try to make API calls to crates.io for local crates

  This shaves 16 seconds off the build, and also avoids some unnecessary
  network calls.

Note that this calls `cargo fetch -Zbuild-std` unconditionally just
because it was easier; I doubt it'll cause any trouble but I can change
it if you like.

Author: jyn514

Cargo.lock

@@ -11,6 +11,7 @@ edition = "2021"
 [workspace]
 exclude = [
     "ignored",
+    "tests",
     ".workspace",
     ".rustwide-docker",
 ]
@@ -30,7 +31,7 @@ tracing-subscriber = { version = "0.3.16", default-features = false, features =
 tracing-log = "0.1.3"
 regex = "1"
 clap = { version = "4.0.22", features = [ "derive" ] }
-crates-index = { version = "0.18.5", optional = true }
+crates-index = { version = "0.19", optional = true }
 rayon = "1.6.1"
 num_cpus = "1.15.0"
 crates-index-diff = { version = "17.0.0", features = [ "max-performance" ]}
@@ -52,7 +53,7 @@ toml = "0.7.2"
 schemamama = "0.3"
 schemamama_postgres = "0.3"
 prometheus = { version = "0.13.0", default-features = false }
-rustwide = { version = "0.15.0", features = ["unstable-toolchain-ci"] }
+rustwide = { version = "0.16.0", features = ["unstable-toolchain-ci", "unstable"] }
 mime_guess = "2"
 zstd = "0.12.0"
 hostname = "0.3.1"

Cargo.toml

@@ -5,7 +5,6 @@ use crate::db::{
 };
 use crate::docbuilder::{crates::crates_from_path, Limits};
 use crate::error::Result;
-use crate::index::api::ReleaseData;
 use crate::repositories::RepositoryStatsUpdater;
 use crate::storage::{rustdoc_archive_path, source_archive_path};
 use crate::utils::{
@@ -16,7 +15,7 @@ use crate::RUSTDOC_STATIC_STORAGE_PREFIX;
 use crate::{db::blacklist::is_blacklisted, utils::MetadataPackage};
 use crate::{Config, Context, Index, Metrics, Storage};
 use anyhow::{anyhow, bail, Context as _, Error};
-use docsrs_metadata::{Metadata, DEFAULT_TARGETS, HOST_TARGET};
+use docsrs_metadata::{BuildTargets, Metadata, DEFAULT_TARGETS, HOST_TARGET};
 use failure::Error as FailureError;
 use postgres::Client;
 use regex::Regex;
@@ -367,6 +366,7 @@ impl RustwideBuilder {
         let mut build_dir = self.workspace.build_dir(&format!("{name}-{version}"));
         build_dir.purge().map_err(FailureError::compat)?;
 
+        let is_local = matches!(kind, PackageKind::Local(_));
         let krate = match kind {
             PackageKind::Local(path) => Crate::local(path),
             PackageKind::CratesIo => Crate::crates_io(name, version),
@@ -383,16 +383,19 @@ impl RustwideBuilder {
         let successful = build_dir
             .build(&self.toolchain, &krate, self.prepare_sandbox(&limits))
             .run(|build| {
-                (|| -> Result<bool> {
-                    use docsrs_metadata::BuildTargets;
+                let metadata = Metadata::from_crate_root(build.host_source_dir())?;
+                let BuildTargets {
+                    default_target,
+                    other_targets,
+                } = metadata.targets(self.config.include_default_targets);
+                let mut targets = vec![default_target];
+                targets.extend(&other_targets);
+                // Fetch this before we enter the sandbox, so networking isn't blocked.
+                build.fetch_build_std_dependencies(&targets)?;
 
+                (|| -> Result<bool> {
                     let mut has_docs = false;
                     let mut successful_targets = Vec::new();
-                    let metadata = Metadata::from_crate_root(build.host_source_dir())?;
-                    let BuildTargets {
-                        default_target,
-                        other_targets,
-                    } = metadata.targets(self.config.include_default_targets);
 
                     // Perform an initial build
                     let mut res =
@@ -488,19 +491,24 @@ impl RustwideBuilder {
                         self.metrics.non_library_builds.inc();
                     }
 
-                    let release_data = match self
-                        .index
-                        .api()
-                        .get_release_data(name, version)
-                        .with_context(|| {
-                            format!("could not fetch releases-data for {name}-{version}")
-                        }) {
-                        Ok(data) => data,
-                        Err(err) => {
-                            report_error(&err);
-                            ReleaseData::default()
+                    let release_data = if !is_local {
+                        match self
+                            .index
+                            .api()
+                            .get_release_data(name, version)
+                            .with_context(|| {
+                                format!("could not fetch releases-data for {name}-{version}")
+                            }) {
+                            Ok(data) => Some(data),
+                            Err(err) => {
+                                report_error(&err);
+                                None
+                            }
                         }
-                    };
+                    } else {
+                        None
+                    }
+                    .unwrap_or_default();
 
                     let cargo_metadata = res.cargo_metadata.root();
                     let repository = self.get_repo(cargo_metadata)?;
@@ -530,11 +538,13 @@ impl RustwideBuilder {
                     self.storage.store_one(build_log_path, res.build_log)?;
 
                     // Some crates.io crate data is mutable, so we proactively update it during a release
-                    match self.index.api().get_crate_data(name) {
-                        Ok(crate_data) => {
-                            update_crate_data_in_database(&mut conn, name, &crate_data)?
+                    if !is_local {
+                        match self.index.api().get_crate_data(name) {
+                            Ok(crate_data) => {
+                                update_crate_data_in_database(&mut conn, name, &crate_data)?
+                            }
+                            Err(err) => warn!("{:#?}", err),
                         }
-                        Err(err) => warn!("{:#?}", err),
                     }
 
                     if res.result.successful {
@@ -725,14 +735,6 @@ impl RustwideBuilder {
         limits: &Limits,
         mut rustdoc_flags_extras: Vec<String>,
     ) -> Result<Command<'ws, 'pl>> {
-        // If the explicit target is not a tier one target, we need to install it.
-        if !docsrs_metadata::DEFAULT_TARGETS.contains(&target) {
-            // This is a no-op if the target is already installed.
-            self.toolchain
-                .add_target(&self.workspace, target)
-                .map_err(FailureError::compat)?;
-        }
-
         // Add docs.rs specific arguments
         let mut cargo_args = vec![
             "--offline".into(),
@@ -781,6 +783,18 @@ impl RustwideBuilder {
         rustdoc_flags_extras.extend(UNCONDITIONAL_ARGS.iter().map(|&s| s.to_owned()));
         let cargo_args = metadata.cargo_args(&cargo_args, &rustdoc_flags_extras);
 
+        // If the explicit target is not a tier one target, we need to install it.
+        let has_build_std = cargo_args.windows(2).any(|args| {
+            args[0].starts_with("-Zbuild-std")
+                || (args[0] == "-Z" && args[1].starts_with("build-std"))
+        }) || cargo_args.last().unwrap().starts_with("-Zbuild-std");
+        if !docsrs_metadata::DEFAULT_TARGETS.contains(&target) && !has_build_std {
+            // This is a no-op if the target is already installed.
+            self.toolchain
+                .add_target(&self.workspace, target)
+                .map_err(FailureError::compat)?;
+        }
+
         let mut command = build
             .cargo()
             .timeout(Some(limits.timeout()))
@@ -1218,4 +1232,14 @@ mod tests {
             Ok(())
         });
     }
+
+    #[test]
+    #[ignore]
+    fn test_build_std() {
+        wrapper(|env| {
+            assert!(RustwideBuilder::init(env)?
+                .build_local_package(Path::new("tests/crates/build-std"))?);
+            Ok(())
+        })
+    }
 }

src/docbuilder/rustwide_builder.rs

@@ -0,0 +1,12 @@
+[package]
+name = "build-std"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+cargo-args = ["-Zbuild-std=core"]

tests/crates/build-std/Cargo.lock

@@ -0,0 +1,14 @@
+pub fn add(left: usize, right: usize) -> usize {
+    left + right
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn it_works() {
+        let result = add(2, 2);
+        assert_eq!(result, 4);
+    }
+}