Auto merge of #57086 - oli-obk:miri_dist, r=kennytm

Prepare everything for distributing miri via rustup

The next step is to tell rustup about `cargo-miri` in https://github.com/rust-lang/rustup.rs/blob/31935e5f633a5acd3a203d23b61d1556c64a821c/src/rustup/lib.rs#L28 and https://github.com/rust-lang/rustup.rs/blob/1ccd706d1d572c777c5134bd7db7aa1a8df7f278/src/rustup-win-installer/src/lib.rs#L29

Author: bors

.gitignore

@@ -13,6 +13,7 @@ __pycache__/
 .settings/
 .valgrindrc
 .vscode/
+.favorites.json
 /*-*-*-*/
 /*-*-*/
 /Makefile

Cargo.lock

@@ -2260,8 +2260,10 @@ dependencies = [
 name = "rustc-workspace-hack"
 version = "1.0.0"
 dependencies = [
+ "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
  "smallvec 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)",

src/bootstrap/builder.rs

@@ -449,6 +449,7 @@ impl<'a> Builder<'a> {
                 dist::Rls,
                 dist::Rustfmt,
                 dist::Clippy,
+                dist::Miri,
                 dist::LlvmTools,
                 dist::Lldb,
                 dist::Extended,
@@ -461,6 +462,7 @@ impl<'a> Builder<'a> {
                 install::Rls,
                 install::Rustfmt,
                 install::Clippy,
+                install::Miri,
                 install::Analysis,
                 install::Src,
                 install::Rustc

src/bootstrap/dist.rs

@@ -32,6 +32,8 @@ pub fn pkgname(builder: &Builder, component: &str) -> String {
         format!("{}-{}", component, builder.rls_package_vers())
     } else if component == "clippy" {
         format!("{}-{}", component, builder.clippy_package_vers())
+    } else if component == "miri" {
+        format!("{}-{}", component, builder.miri_package_vers())
     } else if component == "rustfmt" {
         format!("{}-{}", component, builder.rustfmt_package_vers())
     } else if component == "llvm-tools" {
@@ -1275,6 +1277,90 @@ impl Step for Clippy {
     }
 }
 
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Miri {
+    pub stage: u32,
+    pub target: Interned<String>,
+}
+
+impl Step for Miri {
+    type Output = Option<PathBuf>;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        run.path("miri")
+    }
+
+    fn make_run(run: RunConfig) {
+        run.builder.ensure(Miri {
+            stage: run.builder.top_stage,
+            target: run.target,
+        });
+    }
+
+    fn run(self, builder: &Builder) -> Option<PathBuf> {
+        let stage = self.stage;
+        let target = self.target;
+        assert!(builder.config.extended);
+
+        builder.info(&format!("Dist miri stage{} ({})", stage, target));
+        let src = builder.src.join("src/tools/miri");
+        let release_num = builder.release_num("miri");
+        let name = pkgname(builder, "miri");
+        let version = builder.miri_info.version(builder, &release_num);
+
+        let tmp = tmpdir(builder);
+        let image = tmp.join("miri-image");
+        drop(fs::remove_dir_all(&image));
+        builder.create_dir(&image);
+
+        // Prepare the image directory
+        // We expect miri to build, because we've exited this step above if tool
+        // state for miri isn't testing.
+        let miri = builder.ensure(tool::Miri {
+            compiler: builder.compiler(stage, builder.config.build),
+            target, extra_features: Vec::new()
+        }).or_else(|| { missing_tool("miri", builder.build.config.missing_tools); None })?;
+        let cargomiri = builder.ensure(tool::CargoMiri {
+            compiler: builder.compiler(stage, builder.config.build),
+            target, extra_features: Vec::new()
+        }).or_else(|| { missing_tool("cargo miri", builder.build.config.missing_tools); None })?;
+
+        builder.install(&miri, &image.join("bin"), 0o755);
+        builder.install(&cargomiri, &image.join("bin"), 0o755);
+        let doc = image.join("share/doc/miri");
+        builder.install(&src.join("README.md"), &doc, 0o644);
+        builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
+        builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
+
+        // Prepare the overlay
+        let overlay = tmp.join("miri-overlay");
+        drop(fs::remove_dir_all(&overlay));
+        t!(fs::create_dir_all(&overlay));
+        builder.install(&src.join("README.md"), &overlay, 0o644);
+        builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
+        builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
+        builder.create(&overlay.join("version"), &version);
+
+        // Generate the installer tarball
+        let mut cmd = rust_installer(builder);
+        cmd.arg("generate")
+           .arg("--product-name=Rust")
+           .arg("--rel-manifest-dir=rustlib")
+           .arg("--success-message=miri-ready-to-serve.")
+           .arg("--image-dir").arg(&image)
+           .arg("--work-dir").arg(&tmpdir(builder))
+           .arg("--output-dir").arg(&distdir(builder))
+           .arg("--non-installed-overlay").arg(&overlay)
+           .arg(format!("--package-name={}-{}", name, target))
+           .arg("--legacy-manifest-dirs=rustlib,cargo")
+           .arg("--component-name=miri-preview");
+
+        builder.run(&mut cmd);
+        Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
+    }
+}
+
 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Rustfmt {
     pub stage: u32,
@@ -1396,6 +1482,7 @@ impl Step for Extended {
         let rls_installer = builder.ensure(Rls { stage, target });
         let llvm_tools_installer = builder.ensure(LlvmTools { stage, target });
         let clippy_installer = builder.ensure(Clippy { stage, target });
+        let miri_installer = builder.ensure(Miri { stage, target });
         let lldb_installer = builder.ensure(Lldb { target });
         let mingw_installer = builder.ensure(Mingw { host: target });
         let analysis_installer = builder.ensure(Analysis {
@@ -1434,6 +1521,7 @@ impl Step for Extended {
         tarballs.push(cargo_installer);
         tarballs.extend(rls_installer.clone());
         tarballs.extend(clippy_installer.clone());
+        tarballs.extend(miri_installer.clone());
         tarballs.extend(rustfmt_installer.clone());
         tarballs.extend(llvm_tools_installer);
         tarballs.extend(lldb_installer);
@@ -1506,6 +1594,9 @@ impl Step for Extended {
             if clippy_installer.is_none() {
                 contents = filter(&contents, "clippy");
             }
+            if miri_installer.is_none() {
+                contents = filter(&contents, "miri");
+            }
             if rustfmt_installer.is_none() {
                 contents = filter(&contents, "rustfmt");
             }
@@ -1546,6 +1637,9 @@ impl Step for Extended {
             if clippy_installer.is_some() {
                 prepare("clippy");
             }
+            if miri_installer.is_some() {
+                prepare("miri");
+            }
 
             // create an 'uninstall' package
             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
@@ -1576,6 +1670,8 @@ impl Step for Extended {
                     "rls-preview".to_string()
                 } else if name == "clippy" {
                     "clippy-preview".to_string()
+                } else if name == "miri" {
+                    "miri-preview".to_string()
                 } else {
                     name.to_string()
                 };
@@ -1595,6 +1691,9 @@ impl Step for Extended {
             if clippy_installer.is_some() {
                 prepare("clippy");
             }
+            if miri_installer.is_some() {
+                prepare("miri");
+            }
             if target.contains("windows-gnu") {
                 prepare("rust-mingw");
             }
@@ -1687,6 +1786,18 @@ impl Step for Extended {
                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
             }
+            if miri_installer.is_some() {
+                builder.run(Command::new(&heat)
+                                .current_dir(&exe)
+                                .arg("dir")
+                                .arg("miri")
+                                .args(&heat_flags)
+                                .arg("-cg").arg("MiriGroup")
+                                .arg("-dr").arg("Miri")
+                                .arg("-var").arg("var.MiriDir")
+                                .arg("-out").arg(exe.join("MiriGroup.wxs"))
+                                .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
+            }
             builder.run(Command::new(&heat)
                             .current_dir(&exe)
                             .arg("dir")
@@ -1732,6 +1843,9 @@ impl Step for Extended {
                 if clippy_installer.is_some() {
                     cmd.arg("-dClippyDir=clippy");
                 }
+                if miri_installer.is_some() {
+                    cmd.arg("-dMiriDir=miri");
+                }
                 if target.contains("windows-gnu") {
                     cmd.arg("-dGccDir=rust-mingw");
                 }
@@ -1750,6 +1864,9 @@ impl Step for Extended {
             if clippy_installer.is_some() {
                 candle("ClippyGroup.wxs".as_ref());
             }
+            if miri_installer.is_some() {
+                candle("MiriGroup.wxs".as_ref());
+            }
             candle("AnalysisGroup.wxs".as_ref());
 
             if target.contains("windows-gnu") {
@@ -1782,6 +1899,9 @@ impl Step for Extended {
             if clippy_installer.is_some() {
                 cmd.arg("ClippyGroup.wixobj");
             }
+            if miri_installer.is_some() {
+                cmd.arg("MiriGroup.wixobj");
+            }
 
             if target.contains("windows-gnu") {
                 cmd.arg("GccGroup.wixobj");
@@ -1867,6 +1987,7 @@ impl Step for HashSign {
         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
         cmd.arg(builder.package_vers(&builder.release_num("rls")));
         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
+        cmd.arg(builder.package_vers(&builder.release_num("miri")));
         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
         cmd.arg(builder.llvm_tools_package_vers());
         cmd.arg(builder.lldb_package_vers());

src/bootstrap/install.rs

@@ -32,6 +32,9 @@ pub fn install_rls(builder: &Builder, stage: u32, host: Interned<String>) {
 pub fn install_clippy(builder: &Builder, stage: u32, host: Interned<String>) {
     install_sh(builder, "clippy", "clippy", stage, Some(host));
 }
+pub fn install_miri(builder: &Builder, stage: u32, host: Interned<String>) {
+    install_sh(builder, "miri", "miri", stage, Some(host));
+}
 
 pub fn install_rustfmt(builder: &Builder, stage: u32, host: Interned<String>) {
     install_sh(builder, "rustfmt", "rustfmt", stage, Some(host));
@@ -217,6 +220,14 @@ install!((self, builder, _config),
             builder.info(&format!("skipping Install clippy stage{} ({})", self.stage, self.target));
         }
     };
+    Miri, "miri", Self::should_build(_config), only_hosts: true, {
+        if builder.ensure(dist::Miri { stage: self.stage, target: self.target }).is_some() ||
+            Self::should_install(builder) {
+            install_miri(builder, self.stage, self.target);
+        } else {
+            builder.info(&format!("skipping Install miri stage{} ({})", self.stage, self.target));
+        }
+    };
     Rustfmt, "rustfmt", Self::should_build(_config), only_hosts: true, {
         if builder.ensure(dist::Rustfmt { stage: self.stage, target: self.target }).is_some() ||
             Self::should_install(builder) {

src/bootstrap/lib.rs

@@ -253,6 +253,7 @@ pub struct Build {
     cargo_info: channel::GitInfo,
     rls_info: channel::GitInfo,
     clippy_info: channel::GitInfo,
+    miri_info: channel::GitInfo,
     rustfmt_info: channel::GitInfo,
     local_rebuild: bool,
     fail_fast: bool,
@@ -374,6 +375,7 @@ impl Build {
         let cargo_info = channel::GitInfo::new(&config, &src.join("src/tools/cargo"));
         let rls_info = channel::GitInfo::new(&config, &src.join("src/tools/rls"));
         let clippy_info = channel::GitInfo::new(&config, &src.join("src/tools/clippy"));
+        let miri_info = channel::GitInfo::new(&config, &src.join("src/tools/miri"));
         let rustfmt_info = channel::GitInfo::new(&config, &src.join("src/tools/rustfmt"));
 
         let mut build = Build {
@@ -396,6 +398,7 @@ impl Build {
             cargo_info,
             rls_info,
             clippy_info,
+            miri_info,
             rustfmt_info,
             cc: HashMap::new(),
             cxx: HashMap::new(),
@@ -1016,6 +1019,11 @@ impl Build {
         self.package_vers(&self.release_num("clippy"))
     }
 
+    /// Returns the value of `package_vers` above for miri
+    fn miri_package_vers(&self) -> String {
+        self.package_vers(&self.release_num("miri"))
+    }
+
     /// Returns the value of `package_vers` above for rustfmt
     fn rustfmt_package_vers(&self) -> String {
         self.package_vers(&self.release_num("rustfmt"))

src/bootstrap/tool.rs

@@ -83,6 +83,7 @@ impl Step for ToolBuild {
                 | "rls"
                 | "cargo"
                 | "clippy-driver"
+                | "miri"
                 => {}
 
                 _ => return,
@@ -218,6 +219,7 @@ pub fn prepare_tool_cargo(
         if path.ends_with("cargo") ||
             path.ends_with("rls") ||
             path.ends_with("clippy") ||
+            path.ends_with("miri") ||
             path.ends_with("rustfmt")
         {
             cargo.env("LIBZ_SYS_STATIC", "1");
@@ -592,6 +594,14 @@ tool_extended!((self, builder),
         });
     };
     Miri, miri, "src/tools/miri", "miri", {};
+    CargoMiri, miri, "src/tools/miri", "cargo-miri", {
+        // Miri depends on procedural macros (serde), which requires a full host
+        // compiler to be available, so we need to depend on that.
+        builder.ensure(compile::Rustc {
+            compiler: self.compiler,
+            target: builder.config.build,
+        });
+    };
     Rls, rls, "src/tools/rls", "rls", {
         let clippy = builder.ensure(Clippy {
             compiler: self.compiler,

src/tools/rustc-workspace-hack/Cargo.toml

@@ -27,18 +27,21 @@ features = [
   "errhandlingapi",
   "jobapi",
   "jobapi2",
+  "knownfolders",
   "lmcons",
   "memoryapi",
   "minschannel",
   "minwinbase",
   "ntsecapi",
   "ntstatus",
+  "objbase",
   "profileapi",
   "processenv",
   "psapi",
   "schannel",
   "securitybaseapi",
   "shellapi",
+  "shlobj",
   "sspi",
   "synchapi",
   "sysinfoapi",
@@ -50,12 +53,15 @@ features = [
 ]
 
 [dependencies]
-curl-sys = { version = "0.4.13", optional = true }
+curl-sys = { version = "0.4.13", features = ["http2", "libnghttp2-sys"], optional = true }
 parking_lot = { version = "0.6", features = ['nightly'] }
 rand = { version = "0.5.5", features = ["i128_support"] }
 serde = { version = "1.0.82", features = ['derive'] }
 serde_json = { version = "1.0.31", features = ["raw_value"] }
 smallvec = { version = "0.6", features = ['union'] }
+scopeguard = { version = "0.3.3", features = ["use_std", "default"]}
+byteorder = { version = "1.2.7", features = ["i128"]}
+
 
 [target.'cfg(not(windows))'.dependencies]
 openssl = { version = "0.10.12", optional = true }