make x.py clippy download and use beta clippy

Author: asquared31415

src/bootstrap/bin/rustc.rs

@@ -37,14 +37,25 @@ fn main() {
         Err(_) => 0,
     };
 
+    if verbose > 1 {
+        eprintln!("target: {target:?}");
+        eprintln!("version: {version:?}");
+    }
+
     // Use a different compiler for build scripts, since there may not yet be a
     // libstd for the real compiler to use. However, if Cargo is attempting to
     // determine the version of the compiler, the real compiler needs to be
     // used. Currently, these two states are differentiated based on whether
     // --target and -vV is/isn't passed.
     let (rustc, libdir) = if target.is_none() && version.is_none() {
+        if verbose > 1 {
+            eprintln!("Using snapshot complier");
+        }
         ("RUSTC_SNAPSHOT", "RUSTC_SNAPSHOT_LIBDIR")
     } else {
+        if verbose > 1 {
+            eprintln!("Using real complier");
+        }
         ("RUSTC_REAL", "RUSTC_LIBDIR")
     };
     let stage = env::var("RUSTC_STAGE").unwrap_or_else(|_| {
@@ -75,6 +86,10 @@ fn main() {
                 cmd.arg("-Ztime-passes");
             }
         }
+
+        if crate_name == "build_script_build" {
+            eprintln!("building build scripts using sysroot {:?}", sysroot);
+        }
     }
 
     // Print backtrace in case of ICE
@@ -147,6 +162,15 @@ fn main() {
         cmd.arg("--check-cfg=values(bootstrap)");
     }
 
+    if let Ok(command) = env::var("RUSTC_COMMAND") {
+        if command == "clippy" && target.is_none() {
+            let libdir_string = libdir.to_string_lossy();
+            let (sysroot, _) = libdir_string.rsplit_once('/').unwrap();
+            eprintln!("passing clippy --sysroot {}", sysroot);
+            cmd.arg("--sysroot").arg(&sysroot);
+        }
+    }
+
     if let Ok(map) = env::var("RUSTC_DEBUGINFO_MAP") {
         cmd.arg("--remap-path-prefix").arg(&map);
     }
@@ -220,6 +244,7 @@ fn main() {
             env::join_paths(&dylib_path).unwrap(),
             cmd,
         );
+        eprintln!("{} SYSROOT: {:?}", prefix, env::var("SYSROOT"));
         eprintln!("{} sysroot: {:?}", prefix, sysroot);
         eprintln!("{} libdir: {:?}", prefix, libdir);
     }

src/bootstrap/bootstrap.py

@@ -518,6 +518,8 @@ def download_toolchain(self):
         rustc_channel = self.stage0_compiler.version
         bin_root = self.bin_root()
 
+        tarball_suffix = '.tar.gz' if lzma is None else '.tar.xz'
+
         key = self.stage0_compiler.date
         if self.rustc().startswith(bin_root) and \
                 (not os.path.exists(self.rustc()) or
@@ -602,22 +604,6 @@ def download_toolchain(self):
             with output(self.rustc_stamp()) as rust_stamp:
                 rust_stamp.write(key)
 
-    def _download_component_helper(
-        self, filename, pattern, tarball_suffix, rustc_cache,
-    ):
-        key = self.stage0_compiler.date
-
-        tarball = os.path.join(rustc_cache, filename)
-        if not os.path.exists(tarball):
-            get(
-                self.download_url,
-                "dist/{}/{}".format(key, filename),
-                tarball,
-                self.checksums_sha256,
-                verbose=self.verbose,
-            )
-        unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
-
     def should_fix_bins_and_dylibs(self):
         """Whether or not `fix_bin_or_dylib` needs to be run; can only be True
         on NixOS.
@@ -740,6 +726,27 @@ def rustc_stamp(self):
         """
         return os.path.join(self.bin_root(), '.rustc-stamp')
 
+    def rustfmt_stamp(self):
+        """Return the path for .rustfmt-stamp
+
+        >>> rb = RustBuild()
+        >>> rb.build_dir = "build"
+        >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
+        True
+        """
+        return os.path.join(self.bin_root(), '.rustfmt-stamp')
+
+    def clippy_stamp(self):
+        """Return the path for .clippy-stamp
+
+        >>> rb = RustBuild()
+        >>> rb.build_dir = "build"
+        >>> rb.clippy_stamp() == os.path.join("build", "stage0", ".clippy-stamp")
+        True
+        """
+        return os.path.join(self.bin_root(), '.clippy-stamp')
+
+
     def program_out_of_date(self, stamp_path, key):
         """Check if the given program stamp is out of date"""
         if not os.path.exists(stamp_path) or self.clean:
@@ -810,6 +817,10 @@ def rustc(self):
         """Return config path for rustc"""
         return self.program_config('rustc')
 
+    def clippy(self):
+        """Return config path for clippy"""
+        return self.program_config('cargo-clippy')
+
     def program_config(self, program):
         """Return config path for the given program at the given stage
 

src/bootstrap/builder.rs

@@ -13,7 +13,7 @@ use std::process::Command;
 use std::time::{Duration, Instant};
 
 use crate::cache::{Cache, Interned, INTERNER};
-use crate::config::{SplitDebuginfo, TargetSelection};
+use crate::config::{SplitDebuginfo, TargetSelection, DryRun};
 use crate::doc;
 use crate::flags::{Color, Subcommand};
 use crate::install;
@@ -1171,7 +1171,12 @@ impl<'a> Builder<'a> {
         target: TargetSelection,
         cmd: &str,
     ) -> Command {
-        let mut cargo = Command::new(&self.initial_cargo);
+        let mut cargo = if cmd == "clippy" {
+            Command::new(self.initial_rustc.parent().unwrap().join("cargo-clippy"))
+        } else {
+            Command::new(&self.initial_cargo)
+        };
+
         // Run cargo from the source root so it can find .cargo/config.
         // This matters when using vendoring and the working directory is outside the repository.
         cargo.current_dir(&self.src);
@@ -1293,6 +1298,24 @@ impl<'a> Builder<'a> {
             compiler.stage
         };
 
+        // We synthetically interpret a stage0 compiler used to build tools as a
+        // "raw" compiler in that it's the exact snapshot we download. Normally
+        // the stage0 build means it uses libraries build by the stage0
+        // compiler, but for tools we just use the precompiled libraries that
+        // we've downloaded
+        let use_snapshot = mode == Mode::ToolBootstrap;
+        assert!(!use_snapshot || stage == 0 || self.local_rebuild);
+
+        let maybe_sysroot = self.sysroot(compiler);
+        let sysroot = if use_snapshot { self.rustc_snapshot_sysroot() } else { &maybe_sysroot };
+        let libdir = self.rustc_libdir(compiler);
+
+        let sysroot_str = sysroot.as_os_str().to_str().expect("sysroot should be UTF-8");
+        if !matches!(self.config.dry_run, DryRun::SelfCheck) {
+            self.verbose_than(0, &format!("using sysroot {sysroot_str}"));
+            self.verbose_than(1, &format!("running cargo with mode {mode:?}"));
+        }
+
         let mut rustflags = Rustflags::new(target);
         if stage != 0 {
             if let Ok(s) = env::var("CARGOFLAGS_NOT_BOOTSTRAP") {
@@ -1310,35 +1333,12 @@ impl<'a> Builder<'a> {
                 // NOTE: this can't be fixed in clippy because we explicitly don't set `RUSTC`,
                 // so it has no way of knowing the sysroot.
                 rustflags.arg("--sysroot");
-                rustflags.arg(
-                    self.sysroot(compiler)
-                        .as_os_str()
-                        .to_str()
-                        .expect("sysroot must be valid UTF-8"),
-                );
+                rustflags.arg(sysroot_str);
                 // Only run clippy on a very limited subset of crates (in particular, not build scripts).
                 cargo.arg("-Zunstable-options");
-                // Explicitly does *not* set `--cfg=bootstrap`, since we're using a nightly clippy.
-                let host_version = Command::new("rustc").arg("--version").output().map_err(|_| ());
-                let output = host_version.and_then(|output| {
-                    if output.status.success() {
-                        Ok(output)
-                    } else {
-                        Err(())
-                    }
-                }).unwrap_or_else(|_| {
-                    eprintln!(
-                        "error: `x.py clippy` requires a host `rustc` toolchain with the `clippy` component"
-                    );
-                    eprintln!("help: try `rustup component add clippy`");
-                    crate::detail_exit_macro!(1);
-                });
-                if !t!(std::str::from_utf8(&output.stdout)).contains("nightly") {
-                    rustflags.arg("--cfg=bootstrap");
-                }
-            } else {
-                rustflags.arg("--cfg=bootstrap");
             }
+
+            rustflags.arg("--cfg=bootstrap");
         }
 
         let use_new_symbol_mangling = match self.config.rust_new_symbol_mangling {
@@ -1470,6 +1470,10 @@ impl<'a> Builder<'a> {
             Mode::Std | Mode::Rustc | Mode::Codegen | Mode::ToolRustc => String::new(),
         };
 
+        if self.jobs() > 1 {
+            //panic!("TESTING: Run with one job only!");
+        }
+
         cargo.arg("-j").arg(self.jobs().to_string());
 
         // FIXME: Temporary fix for https://github.com/rust-lang/cargo/issues/3005
@@ -1520,18 +1524,6 @@ impl<'a> Builder<'a> {
 
         let want_rustdoc = self.doc_tests != DocTests::No;
 
-        // We synthetically interpret a stage0 compiler used to build tools as a
-        // "raw" compiler in that it's the exact snapshot we download. Normally
-        // the stage0 build means it uses libraries build by the stage0
-        // compiler, but for tools we just use the precompiled libraries that
-        // we've downloaded
-        let use_snapshot = mode == Mode::ToolBootstrap;
-        assert!(!use_snapshot || stage == 0 || self.local_rebuild);
-
-        let maybe_sysroot = self.sysroot(compiler);
-        let sysroot = if use_snapshot { self.rustc_snapshot_sysroot() } else { &maybe_sysroot };
-        let libdir = self.rustc_libdir(compiler);
-
         // Clear the output directory if the real rustc we're using has changed;
         // Cargo cannot detect this as it thinks rustc is bootstrap/debug/rustc.
         //
@@ -1554,6 +1546,11 @@ impl<'a> Builder<'a> {
             .env("RUSTBUILD_NATIVE_DIR", self.native_dir(target))
             .env("RUSTC_REAL", self.rustc(compiler))
             .env("RUSTC_STAGE", stage.to_string())
+
+            // set for clippy to know the sysroot
+            .env("SYSROOT", &sysroot)
+            .env("RUSTC_COMMAND", cmd)
+
             .env("RUSTC_SYSROOT", &sysroot)
             .env("RUSTC_LIBDIR", &libdir)
             .env("RUSTDOC", self.bootstrap_out.join("rustdoc"))
@@ -1567,11 +1564,8 @@ impl<'a> Builder<'a> {
             )
             .env("RUSTC_ERROR_METADATA_DST", self.extended_error_dir())
             .env("RUSTC_BREAK_ON_ICE", "1");
-        // Clippy support is a hack and uses the default `cargo-clippy` in path.
-        // Don't override RUSTC so that the `cargo-clippy` in path will be run.
-        if cmd != "clippy" {
-            cargo.env("RUSTC", self.bootstrap_out.join("rustc"));
-        }
+
+        cargo.env("RUSTC", self.bootstrap_out.join("rustc"));
 
         // Dealing with rpath here is a little special, so let's go into some
         // detail. First off, `-rpath` is a linker option on Unix platforms

src/tools/bump-stage0/src/main.rs

@@ -4,7 +4,7 @@ use indexmap::IndexMap;
 use std::collections::HashMap;
 
 const PATH: &str = "src/stage0.json";
-const COMPILER_COMPONENTS: &[&str] = &["rustc", "rust-std", "cargo"];
+const COMPILER_COMPONENTS: &[&str] = &["rustc", "rust-std", "cargo", "clippy-preview"];
 const RUSTFMT_COMPONENTS: &[&str] = &["rustfmt-preview", "rustc"];
 
 struct Tool {