Merge pull request #66 from jyn514/build-std

Fetch dependencies for `-Zbuild-std` before entering the sandbox

Author: pietroalbini

CHANGELOG.md

@@ -5,6 +5,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- New method `Build::fetch_build_std_dependencies` for using `-Zbuild-std` within the sandbox when
+  networking is disabled. Previously, this would try to fetch the standard library sources, which
+  would error when networking was blocked.
+
 ## [0.15.2] - 2022-11-08
 
 ### Changed

Cargo.toml

@@ -49,4 +49,5 @@ windows-sys = {version = "0.36.1", features = ["Win32_Foundation", "Win32_System
 
 [dev-dependencies]
 env_logger = "0.8"
+rand = "0.8.5"
 tiny_http = "0.8.0"

src/build.rs

@@ -314,4 +314,20 @@ impl<'ws> Build<'ws> {
     pub fn host_target_dir(&self) -> PathBuf {
         self.dir.target_dir()
     }
+
+    /// Pre-fetching the dependencies for `-Z build-std` outside the sandbox.
+    ///
+    /// When this function is called, it is possible to use `-Zbuild-std` inside
+    /// the sandbox to build the standard library from source even when
+    /// networking is disabled.
+    #[cfg(any(feature = "unstable", doc))]
+    #[cfg_attr(docs_rs, doc(cfg(feature = "unstable")))]
+    pub fn fetch_build_std_dependencies(&self, targets: &[&str]) -> Result<(), Error> {
+        crate::prepare::fetch_deps(
+            &self.dir.workspace,
+            self.toolchain,
+            &self.host_source_dir(),
+            targets,
+        )
+    }
 }

src/prepare.rs

@@ -138,21 +138,40 @@ impl<'a> Prepare<'a> {
     }
 
     fn fetch_deps(&mut self) -> Result<(), Error> {
-        let mut missing_deps = false;
-        let res = Command::new(self.workspace, self.toolchain.cargo())
-            .args(&["fetch", "--manifest-path", "Cargo.toml"])
-            .cd(self.source_dir)
-            .process_lines(&mut |line, _| {
-                if line.contains("failed to load source for dependency") {
-                    missing_deps = true;
-                }
-            })
-            .run();
-        match res {
-            Ok(_) => Ok(()),
-            Err(_) if missing_deps => Err(PrepareError::MissingDependencies.into()),
-            err => err.map_err(|e| e.into()),
-        }
+        fetch_deps(self.workspace, self.toolchain, self.source_dir, &[])
+    }
+}
+
+pub(crate) fn fetch_deps(
+    workspace: &Workspace,
+    toolchain: &Toolchain,
+    source_dir: &Path,
+    fetch_build_std_targets: &[&str],
+) -> Result<(), Error> {
+    let mut missing_deps = false;
+    let mut cmd = Command::new(workspace, toolchain.cargo())
+        .args(&["fetch", "--manifest-path", "Cargo.toml"])
+        .cd(source_dir);
+    // Pass `-Zbuild-std` in case a build in the sandbox wants to use it;
+    // build-std has to have the source for libstd's dependencies available.
+    if !fetch_build_std_targets.is_empty() {
+        toolchain.add_component(workspace, "rust-src")?;
+        cmd = cmd.args(&["-Zbuild-std"]).env("RUSTC_BOOTSTRAP", "1");
+    }
+    for target in fetch_build_std_targets {
+        cmd = cmd.args(&["--target", target]);
+    }
+    let res = cmd
+        .process_lines(&mut |line, _| {
+            if line.contains("failed to load source for dependency") {
+                missing_deps = true;
+            }
+        })
+        .run();
+    match res {
+        Ok(_) => Ok(()),
+        Err(_) if missing_deps => Err(PrepareError::MissingDependencies.into()),
+        err => err.map_err(|e| e.into()),
     }
 }
 

tests/buildtest/mod.rs

@@ -26,6 +26,37 @@ fn test_hello_world() {
     });
 }
 
+#[test]
+#[cfg(feature = "unstable")]
+fn test_fetch_build_std() {
+    use std::path::Path;
+
+    let target_file = Path::new(env!("OUT_DIR")).join("target");
+    let target = std::fs::read_to_string(target_file).unwrap();
+
+    runner::run("hello-world", |run| {
+        run.run(SandboxBuilder::new().enable_networking(false), |build| {
+            build.fetch_build_std_dependencies(&vec![target.as_str()])?;
+            let storage = rustwide::logging::LogStorage::new(LevelFilter::Info);
+            rustwide::logging::capture(&storage, || -> Result<_, Error> {
+                build
+                    .cargo()
+                    .env("RUSTC_BOOTSTRAP", "1")
+                    .args(&["run", "-Zbuild-std", "--target", &target])
+                    .run()?;
+                Ok(())
+            })?;
+
+            assert!(storage.to_string().contains("[stdout] Hello, world!\n"));
+            assert!(storage
+                .to_string()
+                .contains("[stdout] Hello, world again!\n"));
+            Ok(())
+        })?;
+        Ok(())
+    });
+}
+
 #[test]
 fn path_based_patch() {
     runner::run("path-based-patch", |run| {

tests/buildtest/runner.rs

@@ -1,4 +1,5 @@
 use failure::Error;
+use rand::{distributions::Alphanumeric, Rng};
 use rustwide::{cmd::SandboxBuilder, Build, BuildBuilder, Crate, Toolchain, Workspace};
 use std::path::Path;
 
@@ -40,7 +41,16 @@ impl Runner {
         sandbox: SandboxBuilder,
         f: impl FnOnce(BuildBuilder) -> Result<T, Error>,
     ) -> Result<T, Error> {
-        let mut dir = self.workspace.build_dir(&self.crate_name);
+        // Use a random string at the end to avoid conflicts if multiple tests use the same source crate.
+        let suffix: String = rand::thread_rng()
+            .sample_iter(&Alphanumeric)
+            .take(10)
+            .map(char::from)
+            .collect();
+
+        let mut dir = self
+            .workspace
+            .build_dir(&format!("{}-{suffix}", &self.crate_name));
         dir.purge()?;
         f(dir.build(&self.toolchain, &self.krate, sandbox))
     }