Merge pull request #1341 from bjorn3/build_sysroot_cleanup

Refactor sysroot building

Author: bjorn3

.github/workflows/main.yml

@@ -77,7 +77,6 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
-        rustup target add x86_64-pc-windows-gnu
 
     - name: Install AArch64 toolchain and qemu
       if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'aarch64-unknown-linux-gnu'

build_system/bench.rs

@@ -4,7 +4,7 @@ use std::path::Path;
 
 use super::path::{Dirs, RelPath};
 use super::prepare::GitRepo;
-use super::rustc_info::{get_file_name, get_wrapper_file_name};
+use super::rustc_info::get_file_name;
 use super::utils::{hyperfine_command, is_ci, spawn_and_wait, CargoProject, Compiler};
 
 pub(crate) static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
@@ -51,7 +51,8 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
         .unwrap();
 
     eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
-    let cargo_clif = RelPath::DIST.to_path(dirs).join(get_wrapper_file_name("cargo-clif", "bin"));
+    let cargo_clif =
+        RelPath::DIST.to_path(dirs).join(get_file_name("cargo_clif", "bin").replace('_', "-"));
     let manifest_path = SIMPLE_RAYTRACER.manifest_path(dirs);
     let target_dir = SIMPLE_RAYTRACER.target_dir(dirs);
 

build_system/build_sysroot.rs

@@ -1,18 +1,15 @@
 use std::fs;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::process::{self, Command};
 
 use super::path::{Dirs, RelPath};
-use super::rustc_info::{
-    get_file_name, get_rustc_version, get_toolchain_name, get_wrapper_file_name,
-};
+use super::rustc_info::{get_file_name, get_rustc_version, get_toolchain_name};
 use super::utils::{spawn_and_wait, try_hard_link, CargoProject, Compiler};
 use super::SysrootKind;
 
 static DIST_DIR: RelPath = RelPath::DIST;
 static BIN_DIR: RelPath = RelPath::DIST.join("bin");
 static LIB_DIR: RelPath = RelPath::DIST.join("lib");
-static RUSTLIB_DIR: RelPath = LIB_DIR.join("rustlib");
 
 pub(crate) fn build_sysroot(
     dirs: &Dirs,
@@ -39,145 +36,169 @@ pub(crate) fn build_sysroot(
         LIB_DIR
     }
     .to_path(dirs)
-    .join(get_file_name("rustc_codegen_cranelift", "dylib"));
+    .join(cg_clif_dylib_src.file_name().unwrap());
     try_hard_link(cg_clif_dylib_src, &cg_clif_dylib_path);
 
     // Build and copy rustc and cargo wrappers
+    let wrapper_base_name = get_file_name("____", "bin");
+    let toolchain_name = get_toolchain_name();
     for wrapper in ["rustc-clif", "rustdoc-clif", "cargo-clif"] {
-        let wrapper_name = get_wrapper_file_name(wrapper, "bin");
+        let wrapper_name = wrapper_base_name.replace("____", wrapper);
 
         let mut build_cargo_wrapper_cmd = Command::new(&bootstrap_host_compiler.rustc);
         build_cargo_wrapper_cmd
-            .env("TOOLCHAIN_NAME", get_toolchain_name())
+            .env("TOOLCHAIN_NAME", toolchain_name.clone())
             .arg(RelPath::SCRIPTS.to_path(dirs).join(&format!("{wrapper}.rs")))
             .arg("-o")
             .arg(DIST_DIR.to_path(dirs).join(wrapper_name))
-            .arg("-g");
+            .arg("-Cstrip=debuginfo");
         spawn_and_wait(build_cargo_wrapper_cmd);
     }
 
-    let default_sysroot = super::rustc_info::get_default_sysroot(&bootstrap_host_compiler.rustc);
+    let host = build_sysroot_for_triple(
+        dirs,
+        channel,
+        bootstrap_host_compiler.clone(),
+        &cg_clif_dylib_path,
+        sysroot_kind,
+    );
+    host.install_into_sysroot(&DIST_DIR.to_path(dirs));
 
-    let host_rustlib_lib =
-        RUSTLIB_DIR.to_path(dirs).join(&bootstrap_host_compiler.triple).join("lib");
-    let target_rustlib_lib = RUSTLIB_DIR.to_path(dirs).join(&target_triple).join("lib");
-    fs::create_dir_all(&host_rustlib_lib).unwrap();
-    fs::create_dir_all(&target_rustlib_lib).unwrap();
-
-    if target_triple == "x86_64-pc-windows-gnu" {
-        if !default_sysroot.join("lib").join("rustlib").join(&target_triple).join("lib").exists() {
-            eprintln!(
-                "The x86_64-pc-windows-gnu target needs to be installed first before it is possible \
-                to compile a sysroot for it.",
-            );
-            process::exit(1);
-        }
-        for file in fs::read_dir(
-            default_sysroot.join("lib").join("rustlib").join(&target_triple).join("lib"),
+    if !is_native {
+        build_sysroot_for_triple(
+            dirs,
+            channel,
+            {
+                let mut bootstrap_target_compiler = bootstrap_host_compiler.clone();
+                bootstrap_target_compiler.triple = target_triple.clone();
+                bootstrap_target_compiler.set_cross_linker_and_runner();
+                bootstrap_target_compiler
+            },
+            &cg_clif_dylib_path,
+            sysroot_kind,
         )
-        .unwrap()
-        {
-            let file = file.unwrap().path();
-            if file.extension().map_or(true, |ext| ext.to_str().unwrap() != "o") {
-                continue; // only copy object files
-            }
-            try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap()));
-        }
+        .install_into_sysroot(&DIST_DIR.to_path(dirs));
     }
 
-    match sysroot_kind {
-        SysrootKind::None => {} // Nothing to do
-        SysrootKind::Llvm => {
-            for file in fs::read_dir(
-                default_sysroot
-                    .join("lib")
-                    .join("rustlib")
-                    .join(&bootstrap_host_compiler.triple)
-                    .join("lib"),
-            )
-            .unwrap()
-            {
-                let file = file.unwrap().path();
-                let file_name_str = file.file_name().unwrap().to_str().unwrap();
-                if (file_name_str.contains("rustc_")
-                    && !file_name_str.contains("rustc_std_workspace_")
-                    && !file_name_str.contains("rustc_demangle"))
-                    || file_name_str.contains("chalk")
-                    || file_name_str.contains("tracing")
-                    || file_name_str.contains("regex")
-                {
-                    // These are large crates that are part of the rustc-dev component and are not
-                    // necessary to run regular programs.
-                    continue;
-                }
-                try_hard_link(&file, host_rustlib_lib.join(file.file_name().unwrap()));
-            }
-
-            if !is_native {
-                for file in fs::read_dir(
-                    default_sysroot.join("lib").join("rustlib").join(&target_triple).join("lib"),
-                )
-                .unwrap()
-                {
-                    let file = file.unwrap().path();
-                    try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap()));
-                }
-            }
-        }
-        SysrootKind::Clif => {
-            build_clif_sysroot_for_triple(
-                dirs,
-                channel,
-                bootstrap_host_compiler.clone(),
-                &cg_clif_dylib_path,
-            );
-
-            if !is_native {
-                build_clif_sysroot_for_triple(
-                    dirs,
-                    channel,
-                    {
-                        let mut bootstrap_target_compiler = bootstrap_host_compiler.clone();
-                        bootstrap_target_compiler.triple = target_triple.clone();
-                        bootstrap_target_compiler.set_cross_linker_and_runner();
-                        bootstrap_target_compiler
-                    },
-                    &cg_clif_dylib_path,
-                );
-            }
-
-            // Copy std for the host to the lib dir. This is necessary for the jit mode to find
-            // libstd.
-            for file in fs::read_dir(host_rustlib_lib).unwrap() {
-                let file = file.unwrap().path();
-                let filename = file.file_name().unwrap().to_str().unwrap();
-                if filename.contains("std-") && !filename.contains(".rlib") {
-                    try_hard_link(&file, LIB_DIR.to_path(dirs).join(file.file_name().unwrap()));
-                }
-            }
+    // Copy std for the host to the lib dir. This is necessary for the jit mode to find
+    // libstd.
+    for lib in host.libs {
+        let filename = lib.file_name().unwrap().to_str().unwrap();
+        if filename.contains("std-") && !filename.contains(".rlib") {
+            try_hard_link(&lib, LIB_DIR.to_path(dirs).join(lib.file_name().unwrap()));
         }
     }
 
-    let mut target_compiler = Compiler::clif_with_triple(&dirs, target_triple);
+    let mut target_compiler = {
+        let dirs: &Dirs = &dirs;
+        let rustc_clif =
+            RelPath::DIST.to_path(&dirs).join(wrapper_base_name.replace("____", "rustc-clif"));
+        let rustdoc_clif =
+            RelPath::DIST.to_path(&dirs).join(wrapper_base_name.replace("____", "rustdoc-clif"));
+
+        Compiler {
+            cargo: bootstrap_host_compiler.cargo.clone(),
+            rustc: rustc_clif.clone(),
+            rustdoc: rustdoc_clif.clone(),
+            rustflags: String::new(),
+            rustdocflags: String::new(),
+            triple: target_triple,
+            runner: vec![],
+        }
+    };
     if !is_native {
         target_compiler.set_cross_linker_and_runner();
     }
     target_compiler
 }
 
+struct SysrootTarget {
+    triple: String,
+    libs: Vec<PathBuf>,
+}
+
+impl SysrootTarget {
+    fn install_into_sysroot(&self, sysroot: &Path) {
+        if self.libs.is_empty() {
+            return;
+        }
+
+        let target_rustlib_lib = sysroot.join("lib").join("rustlib").join(&self.triple).join("lib");
+        fs::create_dir_all(&target_rustlib_lib).unwrap();
+
+        for lib in &self.libs {
+            try_hard_link(lib, target_rustlib_lib.join(lib.file_name().unwrap()));
+        }
+    }
+}
+
 pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot");
 pub(crate) static BUILD_SYSROOT: RelPath = RelPath::DOWNLOAD.join("sysroot");
 pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = BUILD_SYSROOT.join("rustc_version");
 pub(crate) static SYSROOT_SRC: RelPath = BUILD_SYSROOT.join("sysroot_src");
 pub(crate) static STANDARD_LIBRARY: CargoProject =
     CargoProject::new(&BUILD_SYSROOT, "build_sysroot");
+pub(crate) static RTSTARTUP_SYSROOT: RelPath = RelPath::BUILD.join("rtstartup");
 
+#[must_use]
+fn build_sysroot_for_triple(
+    dirs: &Dirs,
+    channel: &str,
+    compiler: Compiler,
+    cg_clif_dylib_path: &Path,
+    sysroot_kind: SysrootKind,
+) -> SysrootTarget {
+    match sysroot_kind {
+        SysrootKind::None => build_rtstartup(dirs, &compiler)
+            .unwrap_or(SysrootTarget { triple: compiler.triple, libs: vec![] }),
+        SysrootKind::Llvm => build_llvm_sysroot_for_triple(compiler),
+        SysrootKind::Clif => {
+            build_clif_sysroot_for_triple(dirs, channel, compiler, &cg_clif_dylib_path)
+        }
+    }
+}
+
+#[must_use]
+fn build_llvm_sysroot_for_triple(compiler: Compiler) -> SysrootTarget {
+    let default_sysroot = super::rustc_info::get_default_sysroot(&compiler.rustc);
+
+    let mut target_libs = SysrootTarget { triple: compiler.triple, libs: vec![] };
+
+    for entry in fs::read_dir(
+        default_sysroot.join("lib").join("rustlib").join(&target_libs.triple).join("lib"),
+    )
+    .unwrap()
+    {
+        let entry = entry.unwrap();
+        if entry.file_type().unwrap().is_dir() {
+            continue;
+        }
+        let file = entry.path();
+        let file_name_str = file.file_name().unwrap().to_str().unwrap();
+        if (file_name_str.contains("rustc_")
+            && !file_name_str.contains("rustc_std_workspace_")
+            && !file_name_str.contains("rustc_demangle"))
+            || file_name_str.contains("chalk")
+            || file_name_str.contains("tracing")
+            || file_name_str.contains("regex")
+        {
+            // These are large crates that are part of the rustc-dev component and are not
+            // necessary to run regular programs.
+            continue;
+        }
+        target_libs.libs.push(file);
+    }
+
+    target_libs
+}
+
+#[must_use]
 fn build_clif_sysroot_for_triple(
     dirs: &Dirs,
     channel: &str,
     mut compiler: Compiler,
     cg_clif_dylib_path: &Path,
-) {
+) -> SysrootTarget {
     match fs::read_to_string(SYSROOT_RUSTC_VERSION.to_path(dirs)) {
         Err(e) => {
             eprintln!("Failed to get rustc version for patched sysroot source: {}", e);
@@ -196,6 +217,14 @@ fn build_clif_sysroot_for_triple(
         }
     }
 
+    let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] };
+
+    if let Some(rtstartup_target_libs) = build_rtstartup(dirs, &compiler) {
+        rtstartup_target_libs.install_into_sysroot(&RTSTARTUP_SYSROOT.to_path(dirs));
+
+        target_libs.libs.extend(rtstartup_target_libs.libs);
+    }
+
     let build_dir = STANDARD_LIBRARY.target_dir(dirs).join(&compiler.triple).join(channel);
 
     if !super::config::get_bool("keep_sysroot") {
@@ -209,7 +238,9 @@ fn build_clif_sysroot_for_triple(
     // Build sysroot
     let mut rustflags = " -Zforce-unstable-if-unmarked -Cpanic=abort".to_string();
     rustflags.push_str(&format!(" -Zcodegen-backend={}", cg_clif_dylib_path.to_str().unwrap()));
-    rustflags.push_str(&format!(" --sysroot={}", DIST_DIR.to_path(dirs).to_str().unwrap()));
+    // Necessary for MinGW to find rsbegin.o and rsend.o
+    rustflags
+        .push_str(&format!(" --sysroot={}", RTSTARTUP_SYSROOT.to_path(dirs).to_str().unwrap()));
     if channel == "release" {
         rustflags.push_str(" -Zmir-opt-level=3");
     }
@@ -221,7 +252,6 @@ fn build_clif_sysroot_for_triple(
     build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
     spawn_and_wait(build_cmd);
 
-    // Copy all relevant files to the sysroot
     for entry in fs::read_dir(build_dir.join("deps")).unwrap() {
         let entry = entry.unwrap();
         if let Some(ext) = entry.path().extension() {
@@ -231,9 +261,35 @@ fn build_clif_sysroot_for_triple(
         } else {
             continue;
         };
-        try_hard_link(
-            entry.path(),
-            RUSTLIB_DIR.to_path(dirs).join(&compiler.triple).join("lib").join(entry.file_name()),
-        );
+        target_libs.libs.push(entry.path());
+    }
+
+    target_libs
+}
+
+fn build_rtstartup(dirs: &Dirs, compiler: &Compiler) -> Option<SysrootTarget> {
+    if !compiler.triple.ends_with("windows-gnu") {
+        return None;
+    }
+
+    RTSTARTUP_SYSROOT.ensure_fresh(dirs);
+
+    let rtstartup_src = SYSROOT_SRC.to_path(dirs).join("library").join("rtstartup");
+    let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] };
+
+    for file in ["rsbegin", "rsend"] {
+        let obj = RTSTARTUP_SYSROOT.to_path(dirs).join(format!("{file}.o"));
+        let mut build_rtstartup_cmd = Command::new(&compiler.rustc);
+        build_rtstartup_cmd
+            .arg("--target")
+            .arg(&compiler.triple)
+            .arg("--emit=obj")
+            .arg("-o")
+            .arg(&obj)
+            .arg(rtstartup_src.join(format!("{file}.rs")));
+        spawn_and_wait(build_rtstartup_cmd);
+        target_libs.libs.push(obj.clone());
     }
+
+    Some(target_libs)
 }