Auto merge of rust-lang#144303 - Kobzol:bootstrap-tool-cleanup, r=jieyouxu

Consolidate staging for `rustc_private` tools

This PR continues bootstrap refactoring, this time by consolidating staging for `Mode::ToolRustc` tools. This refactoring was in the critical path of refactoring `test`/`dist`/`clippy`/`doc` steps, and getting rid of the rmeta/rlib sysroot copy, because tools are pervasive and they are being used for a lot of things in bootstrap.

The main idea is to explicitly model the fact that a stage N `Mode::ToolRustc` tool always works with two different compilers:
- Stage N-1 rustc (`build_compiler`) builds stage N rustc (`target_compiler`)
- Rlib artifacts from stage N rustc are copied to the sysroot of stage N-1 rustc
- Stage N-1 rustc builds the (stage N) tool itself, the tool links to the rlib artifacts of the stage N rustc

Before, the code often used `compiler`, which meant sometimes the build compiler, sometimes the target compiler, and sometimes neither (looking at you, `download-rustc`). This is especially annoying when you get to a situation where you have an install step that invokes a dist step that invokes a tool build step, where *some* compiler is being propagated through, without it being clear what does that compiler represent. This refactoring hopefully makes that clearer and more explicit. It also gets rid of a few `builder.ensure(Rustc(...))` calls within bootstrap, which is always nice.

`Rustdoc` needs to be handled a bit specially, because it acts as a compiler itself, I documented that in the changes.

It wasn't practical to do these refactorings in multiple PRs, so I did it all in one PR. The meat of the change is 9ee6d1c.

I tested manually that `x build rustdoc` and `x build miri` still works even with `download-rustc`, although I cannot promise any extra support for `download-rustc`, IMO we will just have to reimplement it from scratch in a different way.

As usually, I did some drive-by refactorings to bootstrap, trying to document and clarify things, add more step metadata and tests.

Since these changes broke Cargo, which was incorrectly using `Mode::ToolRustc`, I also changed cargo to `ToolTarget` in this PR.

Best reviewed commit-by-commit (note that I renamed `link_compiler` to `target_compiler`, in accordance to the rest of bootstrap, in the last commit).

r? `@jieyouxu`

try-job: x86_64-gnu-aux
try-job: x86_64-msvc-ext1

Author: bors

src/bootstrap/src/core/build_steps/compile.rs

@@ -19,7 +19,7 @@ use serde_derive::Deserialize;
 use tracing::{instrument, span};
 
 use crate::core::build_steps::gcc::{Gcc, add_cg_gcc_cargo_flags};
-use crate::core::build_steps::tool::{SourceType, copy_lld_artifacts};
+use crate::core::build_steps::tool::{RustcPrivateCompilers, SourceType, copy_lld_artifacts};
 use crate::core::build_steps::{dist, llvm};
 use crate::core::builder;
 use crate::core::builder::{
@@ -1131,7 +1131,7 @@ impl Step for Rustc {
             cargo.env("RUSTC_BOLT_LINK_FLAGS", "1");
         }
 
-        let _guard = builder.msg_sysroot_tool(
+        let _guard = builder.msg_rustc_tool(
             Kind::Build,
             build_compiler.stage,
             format_args!("compiler artifacts{}", crate_description(&self.crates)),
@@ -1544,9 +1544,8 @@ impl Step for RustcLink {
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct CodegenBackend {
-    pub target: TargetSelection,
-    pub compiler: Compiler,
-    pub backend: CodegenBackendKind,
+    compilers: RustcPrivateCompilers,
+    backend: CodegenBackendKind,
 }
 
 fn needs_codegen_config(run: &RunConfig<'_>) -> bool {
@@ -1610,8 +1609,11 @@ impl Step for CodegenBackend {
             }
 
             run.builder.ensure(CodegenBackend {
-                target: run.target,
-                compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()),
+                compilers: RustcPrivateCompilers::new(
+                    run.builder,
+                    run.builder.top_stage,
+                    run.target,
+                ),
                 backend: backend.clone(),
             });
         }
@@ -1624,20 +1626,17 @@ impl Step for CodegenBackend {
             name = "CodegenBackend::run",
             skip_all,
             fields(
-                compiler = ?self.compiler,
-                target = ?self.target,
-                backend = ?self.target,
+                compilers = ?self.compilers,
+                backend = ?self.backend,
             ),
         ),
     )]
     fn run(self, builder: &Builder<'_>) {
-        let compiler = self.compiler;
-        let target = self.target;
         let backend = self.backend;
+        let target = self.compilers.target();
+        let build_compiler = self.compilers.build_compiler();
 
-        builder.ensure(Rustc::new(compiler, target));
-
-        if builder.config.keep_stage.contains(&compiler.stage) {
+        if builder.config.keep_stage.contains(&build_compiler.stage) {
             trace!("`keep-stage` requested");
             builder.info(
                 "WARNING: Using a potentially old codegen backend. \
@@ -1648,17 +1647,11 @@ impl Step for CodegenBackend {
             return;
         }
 
-        let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
-        if compiler_to_use != compiler {
-            builder.ensure(CodegenBackend { compiler: compiler_to_use, target, backend });
-            return;
-        }
-
-        let out_dir = builder.cargo_out(compiler, Mode::Codegen, target);
+        let out_dir = builder.cargo_out(build_compiler, Mode::Codegen, target);
 
         let mut cargo = builder::Cargo::new(
             builder,
-            compiler,
+            build_compiler,
             Mode::Codegen,
             SourceType::InTree,
             target,
@@ -1679,8 +1672,13 @@ impl Step for CodegenBackend {
 
         let tmp_stamp = BuildStamp::new(&out_dir).with_prefix("tmp");
 
-        let _guard =
-            builder.msg_build(compiler, format_args!("codegen backend {}", backend.name()), target);
+        let _guard = builder.msg_rustc_tool(
+            Kind::Build,
+            build_compiler.stage,
+            format_args!("codegen backend {}", backend.name()),
+            build_compiler.host,
+            target,
+        );
         let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false, false);
         if builder.config.dry_run() {
             return;
@@ -1700,10 +1698,17 @@ impl Step for CodegenBackend {
                 f.display()
             );
         }
-        let stamp = build_stamp::codegen_backend_stamp(builder, compiler, target, &backend);
+        let stamp = build_stamp::codegen_backend_stamp(builder, build_compiler, target, &backend);
         let codegen_backend = codegen_backend.to_str().unwrap();
         t!(stamp.add_stamp(codegen_backend).write());
     }
+
+    fn metadata(&self) -> Option<StepMetadata> {
+        Some(
+            StepMetadata::build(&self.backend.crate_name(), self.compilers.target())
+                .built_by(self.compilers.build_compiler()),
+        )
+    }
 }
 
 /// Creates the `codegen-backends` folder for a compiler that's about to be
@@ -2190,8 +2195,10 @@ impl Step for Assemble {
                 continue;
             }
             builder.ensure(CodegenBackend {
-                compiler: build_compiler,
-                target: target_compiler.host,
+                compilers: RustcPrivateCompilers::from_build_and_target_compiler(
+                    build_compiler,
+                    target_compiler,
+                ),
                 backend: backend.clone(),
             });
         }

src/bootstrap/src/core/build_steps/dist.rs

@@ -20,7 +20,7 @@ use object::read::archive::ArchiveFile;
 use tracing::instrument;
 
 use crate::core::build_steps::doc::DocumentationFormat;
-use crate::core::build_steps::tool::{self, Tool};
+use crate::core::build_steps::tool::{self, RustcPrivateCompilers, Tool};
 use crate::core::build_steps::vendor::{VENDOR_DIR, Vendor};
 use crate::core::build_steps::{compile, llvm};
 use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step, StepMetadata};
@@ -425,19 +425,20 @@ impl Step for Rustc {
                 .as_ref()
                 .is_none_or(|tools| tools.iter().any(|tool| tool == "rustdoc"))
             {
-                let rustdoc = builder.rustdoc(compiler);
+                let rustdoc = builder.rustdoc_for_compiler(compiler);
                 builder.install(&rustdoc, &image.join("bin"), FileType::Executable);
             }
 
             let ra_proc_macro_srv_compiler =
                 builder.compiler_for(compiler.stage, builder.config.host_target, compiler.host);
-            builder.ensure(compile::Rustc::new(ra_proc_macro_srv_compiler, compiler.host));
+            let compilers = RustcPrivateCompilers::from_build_compiler(
+                builder,
+                ra_proc_macro_srv_compiler,
+                compiler.host,
+            );
 
             if let Some(ra_proc_macro_srv) = builder.ensure_if_default(
-                tool::RustAnalyzerProcMacroSrv {
-                    compiler: ra_proc_macro_srv_compiler,
-                    target: compiler.host,
-                },
+                tool::RustAnalyzerProcMacroSrv::from_compilers(compilers),
                 builder.kind,
             ) {
                 let dst = image.join("libexec");
@@ -1172,7 +1173,7 @@ impl Step for PlainSourceTarball {
 
 #[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)]
 pub struct Cargo {
-    pub compiler: Compiler,
+    pub build_compiler: Compiler,
     pub target: TargetSelection,
 }
 
@@ -1188,7 +1189,7 @@ impl Step for Cargo {
 
     fn make_run(run: RunConfig<'_>) {
         run.builder.ensure(Cargo {
-            compiler: run.builder.compiler_for(
+            build_compiler: run.builder.compiler_for(
                 run.builder.top_stage,
                 run.builder.config.host_target,
                 run.target,
@@ -1198,12 +1199,10 @@ impl Step for Cargo {
     }
 
     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
-        let compiler = self.compiler;
+        let build_compiler = self.build_compiler;
         let target = self.target;
 
-        builder.ensure(compile::Rustc::new(compiler, target));
-
-        let cargo = builder.ensure(tool::Cargo { compiler, target });
+        let cargo = builder.ensure(tool::Cargo::from_build_compiler(build_compiler, target));
         let src = builder.src.join("src/tools/cargo");
         let etc = src.join("src/etc");
 
@@ -1228,7 +1227,7 @@ impl Step for Cargo {
 
 #[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)]
 pub struct RustAnalyzer {
-    pub compiler: Compiler,
+    pub build_compiler: Compiler,
     pub target: TargetSelection,
 }
 
@@ -1244,7 +1243,7 @@ impl Step for RustAnalyzer {
 
     fn make_run(run: RunConfig<'_>) {
         run.builder.ensure(RustAnalyzer {
-            compiler: run.builder.compiler_for(
+            build_compiler: run.builder.compiler_for(
                 run.builder.top_stage,
                 run.builder.config.host_target,
                 run.target,
@@ -1254,12 +1253,11 @@ impl Step for RustAnalyzer {
     }
 
     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
-        let compiler = self.compiler;
         let target = self.target;
+        let compilers =
+            RustcPrivateCompilers::from_build_compiler(builder, self.build_compiler, self.target);
 
-        builder.ensure(compile::Rustc::new(compiler, target));
-
-        let rust_analyzer = builder.ensure(tool::RustAnalyzer { compiler, target });
+        let rust_analyzer = builder.ensure(tool::RustAnalyzer::from_compilers(compilers));
 
         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
         tarball.set_overlay(OverlayKind::RustAnalyzer);
@@ -1270,9 +1268,9 @@ impl Step for RustAnalyzer {
     }
 }
 
-#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)]
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub struct Clippy {
-    pub compiler: Compiler,
+    pub build_compiler: Compiler,
     pub target: TargetSelection,
 }
 
@@ -1288,7 +1286,7 @@ impl Step for Clippy {
 
     fn make_run(run: RunConfig<'_>) {
         run.builder.ensure(Clippy {
-            compiler: run.builder.compiler_for(
+            build_compiler: run.builder.compiler_for(
                 run.builder.top_stage,
                 run.builder.config.host_target,
                 run.target,
@@ -1298,16 +1296,15 @@ impl Step for Clippy {
     }
 
     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
-        let compiler = self.compiler;
         let target = self.target;
-
-        builder.ensure(compile::Rustc::new(compiler, target));
+        let compilers =
+            RustcPrivateCompilers::from_build_compiler(builder, self.build_compiler, target);
 
         // Prepare the image directory
         // We expect clippy to build, because we've exited this step above if tool
         // state for clippy isn't testing.
-        let clippy = builder.ensure(tool::Clippy { compiler, target });
-        let cargoclippy = builder.ensure(tool::CargoClippy { compiler, target });
+        let clippy = builder.ensure(tool::Clippy::from_compilers(compilers));
+        let cargoclippy = builder.ensure(tool::CargoClippy::from_compilers(compilers));
 
         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
         tarball.set_overlay(OverlayKind::Clippy);
@@ -1319,9 +1316,9 @@ impl Step for Clippy {
     }
 }
 
-#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)]
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub struct Miri {
-    pub compiler: Compiler,
+    pub build_compiler: Compiler,
     pub target: TargetSelection,
 }
 
@@ -1337,7 +1334,7 @@ impl Step for Miri {
 
     fn make_run(run: RunConfig<'_>) {
         run.builder.ensure(Miri {
-            compiler: run.builder.compiler_for(
+            build_compiler: run.builder.compiler_for(
                 run.builder.top_stage,
                 run.builder.config.host_target,
                 run.target,
@@ -1354,15 +1351,12 @@ impl Step for Miri {
             return None;
         }
 
-        let compiler = self.compiler;
-        let target = self.target;
-
-        builder.ensure(compile::Rustc::new(compiler, target));
+        let compilers =
+            RustcPrivateCompilers::from_build_compiler(builder, self.build_compiler, self.target);
+        let miri = builder.ensure(tool::Miri::from_compilers(compilers));
+        let cargomiri = builder.ensure(tool::CargoMiri::from_compilers(compilers));
 
-        let miri = builder.ensure(tool::Miri { compiler, target });
-        let cargomiri = builder.ensure(tool::CargoMiri { compiler, target });
-
-        let mut tarball = Tarball::new(builder, "miri", &target.triple);
+        let mut tarball = Tarball::new(builder, "miri", &self.target.triple);
         tarball.set_overlay(OverlayKind::Miri);
         tarball.is_preview(true);
         tarball.add_file(&miri.tool_path, "bin", FileType::Executable);
@@ -1466,9 +1460,9 @@ impl Step for CodegenBackend {
     }
 }
 
-#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)]
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub struct Rustfmt {
-    pub compiler: Compiler,
+    pub build_compiler: Compiler,
     pub target: TargetSelection,
 }
 
@@ -1484,7 +1478,7 @@ impl Step for Rustfmt {
 
     fn make_run(run: RunConfig<'_>) {
         run.builder.ensure(Rustfmt {
-            compiler: run.builder.compiler_for(
+            build_compiler: run.builder.compiler_for(
                 run.builder.top_stage,
                 run.builder.config.host_target,
                 run.target,
@@ -1494,14 +1488,13 @@ impl Step for Rustfmt {
     }
 
     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
-        let compiler = self.compiler;
-        let target = self.target;
+        let compilers =
+            RustcPrivateCompilers::from_build_compiler(builder, self.build_compiler, self.target);
 
-        builder.ensure(compile::Rustc::new(compiler, target));
+        let rustfmt = builder.ensure(tool::Rustfmt::from_compilers(compilers));
+        let cargofmt = builder.ensure(tool::Cargofmt::from_compilers(compilers));
 
-        let rustfmt = builder.ensure(tool::Rustfmt { compiler, target });
-        let cargofmt = builder.ensure(tool::Cargofmt { compiler, target });
-        let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
+        let mut tarball = Tarball::new(builder, "rustfmt", &self.target.triple);
         tarball.set_overlay(OverlayKind::Rustfmt);
         tarball.is_preview(true);
         tarball.add_file(&rustfmt.tool_path, "bin", FileType::Executable);
@@ -1548,7 +1541,7 @@ impl Step for Extended {
         let mut built_tools = HashSet::new();
         macro_rules! add_component {
             ($name:expr => $step:expr) => {
-                if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
+                if let Some(Some(tarball)) = builder.ensure_if_default($step, Kind::Dist) {
                     tarballs.push(tarball);
                     built_tools.insert($name);
                 }
@@ -1568,12 +1561,12 @@ impl Step for Extended {
 
         add_component!("rust-docs" => Docs { host: target });
         add_component!("rust-json-docs" => JsonDocs { host: target });
-        add_component!("cargo" => Cargo { compiler, target });
-        add_component!("rustfmt" => Rustfmt { compiler, target });
-        add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
+        add_component!("cargo" => Cargo { build_compiler: compiler, target });
+        add_component!("rustfmt" => Rustfmt { build_compiler: compiler, target });
+        add_component!("rust-analyzer" => RustAnalyzer { build_compiler: compiler, target });
         add_component!("llvm-components" => LlvmTools { target });
-        add_component!("clippy" => Clippy { compiler, target });
-        add_component!("miri" => Miri { compiler, target });
+        add_component!("clippy" => Clippy { build_compiler: compiler, target });
+        add_component!("miri" => Miri { build_compiler: compiler, target });
         add_component!("analysis" => Analysis { compiler, target });
         add_component!("rustc-codegen-cranelift" => CodegenBackend {
             compiler: builder.compiler(stage, target),