Auto merge of #116196 - onur-ozkan:reorganize-bootstrap-sources, r=Mark-Simulacrum

reorganize/refactor bootstrap codebase

Currently, bootstrap stores everything on the root path, including very large modules, which makes things very hard to scale and adds too much complexity.

This PR has the following objectives:

- Improving scalability.
- Making bootstrap source more understandable for the new contributors(or for everyone).
- Improving the development experience and making maintenance easier for the bootstrap team.

The new source structure:

```
.
├── defaults
│   ├── README.md
│   ├── config.codegen.toml
│   ├── config.compiler.toml
│   ├── config.dist.toml
│   ├── config.library.toml
│   └── config.tools.toml
├── mk
│   └── Makefile.in
├── src
│   ├── bin
│   │   ├── main.rs
│   │   ├── rustc.rs
│   │   ├── rustdoc.rs
│   │   └── sccache-plus-cl.rs
│   ├── core
│   │   ├── build_steps
│   │   │   ├── check.rs
│   │   │   ├── clean.rs
│   │   │   ├── compile.rs
│   │   │   ├── dist.rs
│   │   │   ├── doc.rs
│   │   │   ├── format.rs
│   │   │   ├── install.rs
│   │   │   ├── llvm.rs
│   │   │   ├── mod.rs
│   │   │   ├── run.rs
│   │   │   ├── setup.rs
│   │   │   ├── suggest.rs
│   │   │   ├── synthetic_targets.rs
│   │   │   ├── test.rs
│   │   │   ├── tool.rs
│   │   │   └── toolstate.rs
│   │   ├── config
│   │   │   ├── config.rs
│   │   │   ├── flags.rs
│   │   │   └── mod.rs
│   │   ├── builder.rs
│   │   ├── download.rs
│   │   ├── metadata.rs
│   │   ├── mod.rs
│   │   └── sanity.rs
│   ├── tests
│   │   ├── builder.rs
│   │   ├── config.rs
│   │   └── setup.rs
│   ├── utils
│   │   ├── bin_helpers.rs
│   │   ├── cache.rs
│   │   ├── cc_detect.rs
│   │   ├── channel.rs
│   │   ├── dylib.rs
│   │   ├── helpers.rs
│   │   ├── job.rs
│   │   ├── metrics.rs
│   │   ├── mod.rs
│   │   ├── render_tests.rs
│   │   └── tarball.rs
│   └── lib.rs
├── Cargo.lock
├── Cargo.toml
├── README.md
├── bootstrap.py
├── bootstrap_test.py
├── build.rs
├── configure.py
└── download-ci-llvm-stamp
```

The next step involves:

- Adding more doc-comments to the bootstrap internals (although we already have a decent amount, there is space for improvement).
- Breaking large modules into smaller, more manageable modules.
- Significantly increasing our unit test coverage (which is currently lacking).

This PR should serve as an initial step to make the tasks above much more easier.

r? Mark-Simulacrum

Author: bors

src/bootstrap/Cargo.toml

@@ -9,27 +9,27 @@ default-run = "bootstrap"
 build-metrics = ["sysinfo"]
 
 [lib]
-path = "lib.rs"
+path = "src/lib.rs"
 doctest = false
 
 [[bin]]
 name = "bootstrap"
-path = "bin/main.rs"
+path = "src/bin/main.rs"
 test = false
 
 [[bin]]
 name = "rustc"
-path = "bin/rustc.rs"
+path = "src/bin/rustc.rs"
 test = false
 
 [[bin]]
 name = "rustdoc"
-path = "bin/rustdoc.rs"
+path = "src/bin/rustdoc.rs"
 test = false
 
 [[bin]]
 name = "sccache-plus-cl"
-path = "bin/sccache-plus-cl.rs"
+path = "src/bin/sccache-plus-cl.rs"
 test = false
 
 [dependencies]

src/bootstrap/job.rs

@@ -15,19 +15,24 @@
 //! switching compilers for the bootstrap and for build scripts will probably
 //! never get replaced.
 
-include!("../dylib_util.rs");
-include!("./_helper.rs");
-
 use std::env;
 use std::path::PathBuf;
-use std::process::{exit, Child, Command};
+use std::process::{Child, Command};
 use std::time::Instant;
 
+use dylib_util::{dylib_path, dylib_path_var};
+
+#[path = "../utils/bin_helpers.rs"]
+mod bin_helpers;
+
+#[path = "../utils/dylib.rs"]
+mod dylib_util;
+
 fn main() {
     let args = env::args_os().skip(1).collect::<Vec<_>>();
     let arg = |name| args.windows(2).find(|args| args[0] == name).and_then(|args| args[1].to_str());
 
-    let verbose = parse_rustc_verbose();
+    let verbose = bin_helpers::parse_rustc_verbose();
 
     // Detect whether or not we're a build script depending on whether --target
     // is passed (a bit janky...)

src/bootstrap/bin/main.rs renamed to src/bootstrap/src/bin/main.rs

@@ -5,17 +5,21 @@
 use std::env;
 use std::ffi::OsString;
 use std::path::PathBuf;
-use std::process::{exit, Command};
+use std::process::Command;
 
-include!("../dylib_util.rs");
+use dylib_util::{dylib_path, dylib_path_var};
 
-include!("./_helper.rs");
+#[path = "../utils/bin_helpers.rs"]
+mod bin_helpers;
+
+#[path = "../utils/dylib.rs"]
+mod dylib_util;
 
 fn main() {
     let args = env::args_os().skip(1).collect::<Vec<_>>();
 
-    let stage = parse_rustc_stage();
-    let verbose = parse_rustc_verbose();
+    let stage = bin_helpers::parse_rustc_stage();
+    let verbose = bin_helpers::parse_rustc_verbose();
 
     let rustdoc = env::var_os("RUSTDOC_REAL").expect("RUSTDOC_REAL was not set");
     let libdir = env::var_os("RUSTDOC_LIBDIR").expect("RUSTDOC_LIBDIR was not set");

src/bootstrap/bin/rustc.rs renamed to src/bootstrap/src/bin/rustc.rs

@@ -1,10 +1,12 @@
 //! Implementation of compiling the compiler and standard library, in "check"-based modes.
 
-use crate::builder::{crate_description, Alias, Builder, Kind, RunConfig, ShouldRun, Step};
-use crate::cache::Interned;
-use crate::compile::{add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo};
-use crate::config::TargetSelection;
-use crate::tool::{prepare_tool_cargo, SourceType};
+use crate::core::build_steps::compile::{
+    add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo,
+};
+use crate::core::build_steps::tool::{prepare_tool_cargo, SourceType};
+use crate::core::builder::{crate_description, Alias, Builder, Kind, RunConfig, ShouldRun, Step};
+use crate::core::config::TargetSelection;
+use crate::utils::cache::Interned;
 use crate::INTERNER;
 use crate::{Compiler, Mode, Subcommand};
 use std::path::{Path, PathBuf};
@@ -16,7 +18,7 @@ pub struct Std {
     ///
     /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`].
     ///
-    /// [`compile::Rustc`]: crate::compile::Rustc
+    /// [`compile::Rustc`]: crate::core::build_steps::compile::Rustc
     crates: Interned<Vec<String>>,
 }
 
@@ -193,7 +195,7 @@ pub struct Rustc {
     ///
     /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`].
     ///
-    /// [`compile::Rustc`]: crate::compile::Rustc
+    /// [`compile::Rustc`]: crate::core::build_steps::compile::Rustc
     crates: Interned<Vec<String>>,
 }
 
@@ -237,8 +239,8 @@ impl Step for Rustc {
             // the sysroot for the compiler to find. Otherwise, we're going to
             // fail when building crates that need to generate code (e.g., build
             // scripts and their dependencies).
-            builder.ensure(crate::compile::Std::new(compiler, compiler.host));
-            builder.ensure(crate::compile::Std::new(compiler, target));
+            builder.ensure(crate::core::build_steps::compile::Std::new(compiler, compiler.host));
+            builder.ensure(crate::core::build_steps::compile::Std::new(compiler, target));
         } else {
             builder.ensure(Std::new(target));
         }
@@ -387,7 +389,7 @@ impl Step for RustAnalyzer {
             &["rust-analyzer/in-rust-tree".to_owned()],
         );
 
-        cargo.allow_features(crate::tool::RustAnalyzer::ALLOW_FEATURES);
+        cargo.allow_features(crate::core::build_steps::tool::RustAnalyzer::ALLOW_FEATURES);
 
         // For ./x.py clippy, don't check those targets because
         // linting tests and benchmarks can produce very noisy results

src/bootstrap/bin/rustdoc.rs renamed to src/bootstrap/src/bin/rustdoc.rs

@@ -9,9 +9,9 @@ use std::fs;
 use std::io::{self, ErrorKind};
 use std::path::Path;
 
-use crate::builder::{crate_description, Builder, RunConfig, ShouldRun, Step};
-use crate::cache::Interned;
-use crate::util::t;
+use crate::core::builder::{crate_description, Builder, RunConfig, ShouldRun, Step};
+use crate::utils::cache::Interned;
+use crate::utils::helpers::t;
 use crate::{Build, Compiler, Mode, Subcommand};
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]

src/bootstrap/bin/sccache-plus-cl.rs renamed to src/bootstrap/src/bin/sccache-plus-cl.rs

@@ -19,16 +19,17 @@ use std::str;
 
 use serde_derive::Deserialize;
 
-use crate::builder::crate_description;
-use crate::builder::Cargo;
-use crate::builder::{Builder, Kind, PathSet, RunConfig, ShouldRun, Step, TaskPath};
-use crate::cache::{Interned, INTERNER};
-use crate::config::{DebuginfoLevel, LlvmLibunwind, RustcLto, TargetSelection};
-use crate::dist;
-use crate::llvm;
-use crate::tool::SourceType;
-use crate::util::get_clang_cl_resource_dir;
-use crate::util::{exe, is_debug_info, is_dylib, output, symlink_dir, t, up_to_date};
+use crate::core::build_steps::dist;
+use crate::core::build_steps::llvm;
+use crate::core::build_steps::tool::SourceType;
+use crate::core::builder::crate_description;
+use crate::core::builder::Cargo;
+use crate::core::builder::{Builder, Kind, PathSet, RunConfig, ShouldRun, Step, TaskPath};
+use crate::core::config::{DebuginfoLevel, LlvmLibunwind, RustcLto, TargetSelection};
+use crate::utils::cache::{Interned, INTERNER};
+use crate::utils::helpers::{
+    exe, get_clang_cl_resource_dir, is_debug_info, is_dylib, output, symlink_dir, t, up_to_date,
+};
 use crate::LLVM_TOOLS;
 use crate::{CLang, Compiler, DependencyType, GitRepo, Mode};
 use filetime::FileTime;
@@ -510,7 +511,7 @@ impl Step for StdLink {
         let (libdir, hostdir) = if self.force_recompile && builder.download_rustc() {
             // NOTE: copies part of `sysroot_libdir` to avoid having to add a new `force_recompile` argument there too
             let lib = builder.sysroot_libdir_relative(self.compiler);
-            let sysroot = builder.ensure(crate::compile::Sysroot {
+            let sysroot = builder.ensure(crate::core::build_steps::compile::Sysroot {
                 compiler: self.compiler,
                 force_recompile: self.force_recompile,
             });
@@ -1016,7 +1017,8 @@ pub fn rustc_cargo_env(
     // detected that LLVM is already built and good to go which helps prevent
     // busting caches (e.g. like #71152).
     if builder.config.llvm_enabled() {
-        let building_is_expensive = crate::llvm::prebuilt_llvm_config(builder, target).is_err();
+        let building_is_expensive =
+            crate::core::build_steps::llvm::prebuilt_llvm_config(builder, target).is_err();
         // `top_stage == stage` might be false for `check --stage 1`, if we are building the stage 1 compiler
         let can_skip_build = builder.kind == Kind::Check && builder.top_stage == stage;
         let should_skip_build = building_is_expensive && can_skip_build;
@@ -1684,7 +1686,7 @@ impl Step for Assemble {
             builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe));
             let self_contained_lld_dir = libdir_bin.join("gcc-ld");
             t!(fs::create_dir(&self_contained_lld_dir));
-            let lld_wrapper_exe = builder.ensure(crate::tool::LldWrapper {
+            let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper {
                 compiler: build_compiler,
                 target: target_compiler.host,
             });