Auto merge of #41843 - cuviper:oxidized-installer, r=alexcrichton

Update to the Rusty rust-installer

This updates the rust-installer submodule to the new version written in Rust (rust-lang/rust-installer#62), now moved to `src/tools/rust-installer` and invoked in `dist.rs` as a cargo-based tool command.  All of the former shell-script invocations now invoke the tool, otherwise keeping the same arguments as before.

As a small bonus, `rustc-src` now also uses the same tarball generator, so it gains a smaller `.tar.xz` too.

Fixes #41569.  r? @alexcrichton

Author: bors

.gitmodules

@@ -13,7 +13,7 @@
 	path = src/jemalloc
 	url = https://github.com/rust-lang/jemalloc.git
 [submodule "src/rust-installer"]
-	path = src/rust-installer
+	path = src/tools/rust-installer
 	url = https://github.com/rust-lang/rust-installer.git
 [submodule "src/liblibc"]
 	path = src/liblibc

src/Cargo.lock

@@ -13,6 +13,7 @@ members = [
   "tools/build-manifest",
   "tools/remote-test-client",
   "tools/remote-test-server",
+  "tools/rust-installer",
 ]
 
 # These projects have their own Cargo.lock

src/Cargo.toml

@@ -26,12 +26,6 @@ use std::process::{Command, Stdio};
 
 use build_helper::output;
 
-#[cfg(not(target_os = "solaris"))]
-const SH_CMD: &'static str = "sh";
-// On Solaris, sh is the historical bourne shell, not a POSIX shell, or bash.
-#[cfg(target_os = "solaris")]
-const SH_CMD: &'static str = "bash";
-
 use {Build, Compiler, Mode};
 use channel;
 use util::{cp_r, libdir, is_dylib, cp_filtered, copy, exe};
@@ -55,6 +49,10 @@ pub fn tmpdir(build: &Build) -> PathBuf {
     build.out.join("tmp/dist")
 }
 
+fn rust_installer(build: &Build) -> Command {
+    build.tool_cmd(&Compiler::new(0, &build.config.build), "rust-installer")
+}
+
 /// Builds the `rust-docs` installer component.
 ///
 /// Slurps up documentation from the `stage`'s `host`.
@@ -74,14 +72,14 @@ pub fn docs(build: &Build, stage: u32, host: &str) {
     let src = build.out.join(host).join("doc");
     cp_r(&src, &dst);
 
-    let mut cmd = Command::new(SH_CMD);
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("generate")
        .arg("--product-name=Rust-Documentation")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=Rust-documentation-is-installed.")
-       .arg(format!("--image-dir={}", sanitize_sh(&image)))
-       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg("--image-dir").arg(&image)
+       .arg("--work-dir").arg(&tmpdir(build))
+       .arg("--output-dir").arg(&distdir(build))
        .arg(format!("--package-name={}-{}", name, host))
        .arg("--component-name=rust-docs")
        .arg("--legacy-manifest-dirs=rustlib,cargo")
@@ -124,14 +122,14 @@ pub fn mingw(build: &Build, host: &str) {
        .arg(host);
     build.run(&mut cmd);
 
-    let mut cmd = Command::new(SH_CMD);
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("generate")
        .arg("--product-name=Rust-MinGW")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=Rust-MinGW-is-installed.")
-       .arg(format!("--image-dir={}", sanitize_sh(&image)))
-       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg("--image-dir").arg(&image)
+       .arg("--work-dir").arg(&tmpdir(build))
+       .arg("--output-dir").arg(&distdir(build))
        .arg(format!("--package-name={}-{}", name, host))
        .arg("--component-name=rust-mingw")
        .arg("--legacy-manifest-dirs=rustlib,cargo");
@@ -190,15 +188,15 @@ pub fn rustc(build: &Build, stage: u32, host: &str) {
     }
 
     // Finally, wrap everything up in a nice tarball!
-    let mut cmd = Command::new(SH_CMD);
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("generate")
        .arg("--product-name=Rust")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=Rust-is-ready-to-roll.")
-       .arg(format!("--image-dir={}", sanitize_sh(&image)))
-       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
-       .arg(format!("--non-installed-overlay={}", sanitize_sh(&overlay)))
+       .arg("--image-dir").arg(&image)
+       .arg("--work-dir").arg(&tmpdir(build))
+       .arg("--output-dir").arg(&distdir(build))
+       .arg("--non-installed-overlay").arg(&overlay)
        .arg(format!("--package-name={}-{}", name, host))
        .arg("--component-name=rustc")
        .arg("--legacy-manifest-dirs=rustlib,cargo");
@@ -300,14 +298,14 @@ pub fn std(build: &Build, compiler: &Compiler, target: &str) {
     let src = build.sysroot(compiler).join("lib/rustlib");
     cp_r(&src.join(target), &dst);
 
-    let mut cmd = Command::new(SH_CMD);
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("generate")
        .arg("--product-name=Rust")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=std-is-standing-at-the-ready.")
-       .arg(format!("--image-dir={}", sanitize_sh(&image)))
-       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg("--image-dir").arg(&image)
+       .arg("--work-dir").arg(&tmpdir(build))
+       .arg("--output-dir").arg(&distdir(build))
        .arg(format!("--package-name={}-{}", name, target))
        .arg(format!("--component-name=rust-std-{}", target))
        .arg("--legacy-manifest-dirs=rustlib,cargo");
@@ -356,14 +354,14 @@ pub fn analysis(build: &Build, compiler: &Compiler, target: &str) {
     println!("image_src: {:?}, dst: {:?}", image_src, dst);
     cp_r(&image_src, &dst);
 
-    let mut cmd = Command::new(SH_CMD);
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("generate")
        .arg("--product-name=Rust")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=save-analysis-saved.")
-       .arg(format!("--image-dir={}", sanitize_sh(&image)))
-       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg("--image-dir").arg(&image)
+       .arg("--work-dir").arg(&tmpdir(build))
+       .arg("--output-dir").arg(&distdir(build))
        .arg(format!("--package-name={}-{}", name, target))
        .arg(format!("--component-name=rust-analysis-{}", target))
        .arg("--legacy-manifest-dirs=rustlib,cargo");
@@ -471,13 +469,17 @@ pub fn rust_src(build: &Build) {
     write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes());
 
     // Create plain source tarball
-    let tarball = rust_src_location(build);
+    let mut tarball = rust_src_location(build);
+    tarball.set_extension(""); // strip .gz
+    tarball.set_extension(""); // strip .tar
     if let Some(dir) = tarball.parent() {
         t!(fs::create_dir_all(dir));
     }
-    let mut cmd = Command::new("tar");
-    cmd.arg("-czf").arg(sanitize_sh(&tarball))
-       .arg(&plain_name)
+    let mut cmd = rust_installer(build);
+    cmd.arg("tarball")
+       .arg("--input").arg(&plain_name)
+       .arg("--output").arg(&tarball)
+       .arg("--work-dir=.")
        .current_dir(tmpdir(build));
     build.run(&mut cmd);
 
@@ -521,14 +523,14 @@ pub fn rust_src(build: &Build) {
     }
 
     // Create source tarball in rust-installer format
-    let mut cmd = Command::new(SH_CMD);
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("generate")
        .arg("--product-name=Rust")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=Awesome-Source.")
-       .arg(format!("--image-dir={}", sanitize_sh(&image)))
-       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg("--image-dir").arg(&image)
+       .arg("--work-dir").arg(&tmpdir(build))
+       .arg("--output-dir").arg(&distdir(build))
        .arg(format!("--package-name={}", name))
        .arg("--component-name=rust-src")
        .arg("--legacy-manifest-dirs=rustlib,cargo");
@@ -622,15 +624,15 @@ pub fn cargo(build: &Build, stage: u32, target: &str) {
     t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
 
     // Generate the installer tarball
-    let mut cmd = Command::new("sh");
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("generate")
        .arg("--product-name=Rust")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=Rust-is-ready-to-roll.")
-       .arg(format!("--image-dir={}", sanitize_sh(&image)))
-       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
-       .arg(format!("--non-installed-overlay={}", sanitize_sh(&overlay)))
+       .arg("--image-dir").arg(&image)
+       .arg("--work-dir").arg(&tmpdir(build))
+       .arg("--output-dir").arg(&distdir(build))
+       .arg("--non-installed-overlay").arg(&overlay)
        .arg(format!("--package-name={}-{}", name, target))
        .arg("--component-name=cargo")
        .arg("--legacy-manifest-dirs=rustlib,cargo");
@@ -671,15 +673,15 @@ pub fn rls(build: &Build, stage: u32, target: &str) {
     t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
 
     // Generate the installer tarball
-    let mut cmd = Command::new("sh");
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("generate")
        .arg("--product-name=Rust")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=RLS-ready-to-serve.")
-       .arg(format!("--image-dir={}", sanitize_sh(&image)))
-       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
-       .arg(format!("--non-installed-overlay={}", sanitize_sh(&overlay)))
+       .arg("--image-dir").arg(&image)
+       .arg("--work-dir").arg(&tmpdir(build))
+       .arg("--output-dir").arg(&distdir(build))
+       .arg("--non-installed-overlay").arg(&overlay)
        .arg(format!("--package-name={}-{}", name, target))
        .arg("--component-name=rls")
        .arg("--legacy-manifest-dirs=rustlib,cargo");
@@ -730,29 +732,28 @@ pub fn extended(build: &Build, stage: u32, target: &str) {
     // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
     // the std files during uninstall. To do this ensure that rustc comes
     // before rust-std in the list below.
-    let mut input_tarballs = format!("{},{},{},{},{},{}",
-                                     sanitize_sh(&rustc_installer),
-                                     sanitize_sh(&cargo_installer),
-                                     sanitize_sh(&rls_installer),
-                                     sanitize_sh(&analysis_installer),
-                                     sanitize_sh(&docs_installer),
-                                     sanitize_sh(&std_installer));
+    let mut tarballs = vec![rustc_installer, cargo_installer, rls_installer,
+                            analysis_installer, docs_installer, std_installer];
     if target.contains("pc-windows-gnu") {
-        input_tarballs.push_str(",");
-        input_tarballs.push_str(&sanitize_sh(&mingw_installer));
+        tarballs.push(mingw_installer);
+    }
+    let mut input_tarballs = tarballs[0].as_os_str().to_owned();
+    for tarball in &tarballs[1..] {
+        input_tarballs.push(",");
+        input_tarballs.push(tarball);
     }
 
-    let mut cmd = Command::new(SH_CMD);
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/combine-installers.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("combine")
        .arg("--product-name=Rust")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=Rust-is-ready-to-roll.")
-       .arg(format!("--work-dir={}", sanitize_sh(&work)))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg("--work-dir").arg(&work)
+       .arg("--output-dir").arg(&distdir(build))
        .arg(format!("--package-name={}-{}", pkgname(build, "rust"), target))
        .arg("--legacy-manifest-dirs=rustlib,cargo")
-       .arg(format!("--input-tarballs={}", input_tarballs))
-       .arg(format!("--non-installed-overlay={}", sanitize_sh(&overlay)));
+       .arg("--input-tarballs").arg(input_tarballs)
+       .arg("--non-installed-overlay").arg(&overlay);
     build.run(&mut cmd);
 
     let mut license = String::new();

src/bootstrap/dist.rs

@@ -574,6 +574,10 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .dep(|s| s.name("maybe-clean-tools"))
          .dep(|s| s.name("libstd-tool"))
          .run(move |s| compile::tool(build, s.stage, s.target, "remote-test-client"));
+    rules.build("tool-rust-installer", "src/tools/rust-installer")
+         .dep(|s| s.name("maybe-clean-tools"))
+         .dep(|s| s.name("libstd-tool"))
+         .run(move |s| compile::tool(build, s.stage, s.target, "rust-installer"));
     rules.build("tool-cargo", "src/tools/cargo")
          .host(true)
          .default(build.config.extended)
@@ -704,6 +708,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .host(true)
          .only_host_build(true)
          .default(true)
+         .dep(move |s| tool_rust_installer(build, s))
          .run(move |s| dist::rustc(build, s.stage, s.target));
     rules.dist("dist-std", "src/libstd")
          .dep(move |s| {
@@ -718,10 +723,12 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          })
          .default(true)
          .only_host_build(true)
+         .dep(move |s| tool_rust_installer(build, s))
          .run(move |s| dist::std(build, &s.compiler(), s.target));
     rules.dist("dist-mingw", "path/to/nowhere")
          .default(true)
          .only_host_build(true)
+         .dep(move |s| tool_rust_installer(build, s))
          .run(move |s| {
              if s.target.contains("pc-windows-gnu") {
                  dist::mingw(build, s.target)
@@ -732,21 +739,25 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .host(true)
          .only_build(true)
          .only_host_build(true)
+         .dep(move |s| tool_rust_installer(build, s))
          .run(move |_| dist::rust_src(build));
     rules.dist("dist-docs", "src/doc")
          .default(true)
          .only_host_build(true)
          .dep(|s| s.name("default:doc"))
+         .dep(move |s| tool_rust_installer(build, s))
          .run(move |s| dist::docs(build, s.stage, s.target));
     rules.dist("dist-analysis", "analysis")
          .default(build.config.extended)
          .dep(|s| s.name("dist-std"))
          .only_host_build(true)
+         .dep(move |s| tool_rust_installer(build, s))
          .run(move |s| dist::analysis(build, &s.compiler(), s.target));
     rules.dist("dist-rls", "rls")
          .host(true)
          .only_host_build(true)
          .dep(|s| s.name("tool-rls"))
+         .dep(move |s| tool_rust_installer(build, s))
          .run(move |s| dist::rls(build, s.stage, s.target));
     rules.dist("install", "path/to/nowhere")
          .dep(|s| s.name("default:dist"))
@@ -755,6 +766,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .host(true)
          .only_host_build(true)
          .dep(|s| s.name("tool-cargo"))
+         .dep(move |s| tool_rust_installer(build, s))
          .run(move |s| dist::cargo(build, s.stage, s.target));
     rules.dist("dist-extended", "extended")
          .default(build.config.extended)
@@ -767,6 +779,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .dep(|d| d.name("dist-cargo"))
          .dep(|d| d.name("dist-rls"))
          .dep(|d| d.name("dist-analysis"))
+         .dep(move |s| tool_rust_installer(build, s))
          .run(move |s| dist::extended(build, s.stage, s.target));
 
     rules.dist("dist-sign", "hash-and-sign")
@@ -778,6 +791,14 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
 
     rules.verify();
     return rules;
+
+    /// Helper to depend on a stage0 build-only rust-installer tool.
+    fn tool_rust_installer<'a>(build: &'a Build, step: &Step<'a>) -> Step<'a> {
+        step.name("tool-rust-installer")
+            .host(&build.config.build)
+            .target(&build.config.build)
+            .stage(0)
+    }
 }
 
 #[derive(PartialEq, Eq, Hash, Clone, Debug)]

src/bootstrap/step.rs

@@ -81,12 +81,12 @@ fn filter_dirs(path: &Path) -> bool {
         "src/libbacktrace",
         "src/compiler-rt",
         "src/rustllvm",
-        "src/rust-installer",
         "src/liblibc",
         "src/vendor",
         "src/rt/hoedown",
         "src/tools/cargo",
         "src/tools/rls",
+        "src/tools/rust-installer",
     ];
     skip.iter().any(|p| path.ends_with(p))
 }