Add support for downloading libgccjit.so file to prevent requiring to build it yourself

Author: GuillaumeGomez

compiler/rustc_codegen_gcc/build_system/src/config.rs

@@ -444,15 +444,17 @@ impl ConfigInfo {
             "build_sysroot/sysroot/lib/rustlib/{}/lib",
             self.target_triple,
         ));
-        let ld_library_path = format!(
-            "{target}:{sysroot}:{gcc_path}",
-            target = self.cargo_target_dir,
-            sysroot = sysroot.display(),
-            gcc_path = self.gcc_path,
-        );
-        env.insert("LIBRARY_PATH".to_string(), ld_library_path.clone());
-        env.insert("LD_LIBRARY_PATH".to_string(), ld_library_path.clone());
-        env.insert("DYLD_LIBRARY_PATH".to_string(), ld_library_path);
+        if !use_system_gcc {
+            let ld_library_path = format!(
+                "{target}:{sysroot}:{gcc_path}",
+                target = self.cargo_target_dir,
+                sysroot = sysroot.display(),
+                gcc_path = self.gcc_path,
+            );
+            env.insert("LIBRARY_PATH".to_string(), ld_library_path.clone());
+            env.insert("LD_LIBRARY_PATH".to_string(), ld_library_path.clone());
+            env.insert("DYLD_LIBRARY_PATH".to_string(), ld_library_path);
+        }
 
         // NOTE: To avoid the -fno-inline errors, use /opt/gcc/bin/gcc instead of cc.
         // To do so, add a symlink for cc to /opt/gcc/bin/gcc in our PATH.

config.example.toml

@@ -155,6 +155,13 @@
 # Custom CMake defines to set when building LLVM.
 #build-config = {}
 
+# =============================================================================
+# General configuration options for the GCC backend
+# =============================================================================
+[gcc]
+# Change to `true` if you want to use the version used in Rust CI.
+# download-gccjit = false
+
 # =============================================================================
 # General build configuration options
 # =============================================================================

src/bootstrap/src/core/build_steps/test.rs

@@ -3503,6 +3503,7 @@ impl Step for CodegenGCC {
             // Avoid incremental cache issues when changing rustc
             cargo.env("CARGO_BUILD_INCREMENTAL", "false");
             cargo.rustflag("-Cpanic=abort");
+            cargo.add_rustc_lib_path(builder);
 
             cargo
         };

src/bootstrap/src/core/builder.rs

@@ -21,8 +21,10 @@ use crate::core::config::flags::{Color, Subcommand};
 use crate::core::config::{DryRun, SplitDebuginfo, TargetSelection};
 use crate::prepare_behaviour_dump_dir;
 use crate::utils::cache::Cache;
-use crate::utils::helpers::{self, add_dylib_path, add_link_lib_path, exe, linker_args};
-use crate::utils::helpers::{check_cfg_arg, libdir, linker_flags, output, t, LldThreads};
+use crate::utils::helpers::{
+    self, add_dylib_path, add_link_lib_path, check_cfg_arg, detect_gccjit_sha, exe, libdir,
+    linker_args, linker_flags, output, t, LldThreads,
+};
 use crate::EXTRA_CHECK_CFGS;
 use crate::{Build, CLang, Crate, DocTests, GitRepo, Mode};
 
@@ -1196,6 +1198,10 @@ impl<'a> Builder<'a> {
             let ci_llvm_lib = self.out.join(&*compiler.host.triple).join("ci-llvm").join("lib");
             dylib_dirs.push(ci_llvm_lib);
         }
+        if self.config.gcc_download_gccjit {
+            let libgccjit_path = self.config.libgccjit_folder(&detect_gccjit_sha());
+            dylib_dirs.push(libgccjit_path);
+        }
 
         dylib_dirs
     }

src/bootstrap/src/core/config/config.rs

@@ -250,6 +250,8 @@ pub struct Config {
     pub llvm_ldflags: Option<String>,
     pub llvm_use_libcxx: bool,
 
+    pub gcc_download_gccjit: bool,
+
     // rust codegen options
     pub rust_optimize: RustOptimize,
     pub rust_codegen_units: Option<u32>,
@@ -592,6 +594,7 @@ pub(crate) struct TomlConfig {
     build: Option<Build>,
     install: Option<Install>,
     llvm: Option<Llvm>,
+    gcc: Option<Gcc>,
     rust: Option<Rust>,
     target: Option<HashMap<String, TomlTarget>>,
     dist: Option<Dist>,
@@ -626,7 +629,7 @@ trait Merge {
 impl Merge for TomlConfig {
     fn merge(
         &mut self,
-        TomlConfig { build, install, llvm, rust, dist, target, profile: _, change_id }: Self,
+        TomlConfig { build, install, llvm, rust, dist, target, profile: _, change_id, gcc }: Self,
         replace: ReplaceOpt,
     ) {
         fn do_merge<T: Merge>(x: &mut Option<T>, y: Option<T>, replace: ReplaceOpt) {
@@ -644,6 +647,7 @@ impl Merge for TomlConfig {
         do_merge(&mut self.llvm, llvm, replace);
         do_merge(&mut self.rust, rust, replace);
         do_merge(&mut self.dist, dist, replace);
+        do_merge(&mut self.gcc, gcc, replace);
         assert!(target.is_none(), "merging target-specific config is not currently supported");
     }
 }
@@ -899,6 +903,13 @@ define_config! {
     }
 }
 
+define_config! {
+    /// TOML representation of how the GCC backend is configured.
+    struct Gcc {
+        download_gccjit: Option<bool> = "download-gccjit",
+    }
+}
+
 #[derive(Clone, Debug, Deserialize)]
 #[serde(untagged)]
 pub enum StringOrBool {
@@ -1724,6 +1735,14 @@ impl Config {
             ci_channel.clone_into(&mut config.channel);
         }
 
+        if let Some(gcc) = toml.gcc {
+            let Gcc { download_gccjit } = gcc;
+            config.gcc_download_gccjit = download_gccjit.unwrap_or(false);
+        }
+        if config.gcc_enabled(config.build) {
+            config.maybe_download_gccjit();
+        }
+
         if let Some(llvm) = toml.llvm {
             let Llvm {
                 optimize: optimize_toml,
@@ -2329,6 +2348,18 @@ impl Config {
         self.codegen_backends(target).contains(&"llvm".to_owned())
     }
 
+    pub fn gcc_enabled(&self, target: TargetSelection) -> bool {
+        self.codegen_backends(target).contains(&"gcc".to_owned())
+    }
+
+    pub fn libgccjit_folder(&self, gcc_sha: &str) -> PathBuf {
+        assert!(self.gcc_download_gccjit);
+        let cache_prefix = format!("libgccjit-{gcc_sha}");
+        let cache_dst =
+            self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache"));
+        cache_dst.join(cache_prefix)
+    }
+
     pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind {
         self.target_config
             .get(&target)

src/bootstrap/src/core/download.rs

@@ -12,7 +12,8 @@ use build_helper::ci::CiEnv;
 use build_helper::stage0_parser::VersionMetadata;
 use xz2::bufread::XzDecoder;
 
-use crate::utils::helpers::{check_run, exe, move_file, program_out_of_date};
+use crate::core::config::RustfmtMetadata;
+use crate::utils::helpers::{check_run, detect_gccjit_sha, exe, move_file, program_out_of_date};
 use crate::{core::build_steps::llvm::detect_llvm_sha, utils::helpers::hex_encode};
 use crate::{t, Config};
 
@@ -229,7 +230,7 @@ impl Config {
             tempfile.to_str().unwrap(),
             "--retry",
             "3",
-            "-SRf",
+            "-SRfL",
         ]);
         // Don't print progress in CI; the \r wrapping looks bad and downloads don't take long enough for progress to be useful.
         if CiEnv::is_ci() {
@@ -742,6 +743,38 @@ download-rustc = false
         let llvm_root = self.ci_llvm_root();
         self.unpack(&tarball, &llvm_root, "rust-dev");
     }
+
+    pub(crate) fn maybe_download_gccjit(&self) {
+        if !self.gcc_download_gccjit {
+            return;
+        }
+        self.download_gccjit(&detect_gccjit_sha());
+    }
+
+    fn download_gccjit(&self, gcc_sha: &str) {
+        let help_on_error = "ERROR: failed to download libgccjit
+
+    HELP: There could be two reasons behind this:
+        1) The host triple is not supported for `download-gccjit`.
+        2) Old builds get deleted after a certain time.
+    ";
+        if !self.build.triple.contains("linux") || !self.build.triple.contains("x86_64") {
+            eprintln!("{help_on_error}");
+            return;
+        }
+        let rustc_cache = self.libgccjit_folder(gcc_sha);
+        if !rustc_cache.exists() {
+            t!(fs::create_dir_all(&rustc_cache));
+        }
+
+        let lib_path = rustc_cache.join("libgccjit.so.0");
+        if !lib_path.exists() {
+            let url = format!(
+                "https://github.com/rust-lang/gcc/releases/download/master-{gcc_sha}/libgccjit.so",
+            );
+            self.download_file(&url, &lib_path, help_on_error);
+        }
+    }
 }
 
 fn path_is_dylib(path: &Path) -> bool {

src/bootstrap/src/utils/helpers.rs

@@ -598,3 +598,22 @@ pub fn check_cfg_arg(name: &str, values: Option<&[&str]>) -> String {
     };
     format!("--check-cfg=cfg({name}{next})")
 }
+
+/// This retrieves the gccjit sha we use.
+pub fn detect_gccjit_sha() -> String {
+    // FIXME: This is absolutely ugly. Please find another way to do that.
+    let content = include_str!("../../../ci/docker/host-x86_64/dist-x86_64-linux/build-gccjit.sh");
+    if let Some(commit) = content
+        .lines()
+        .filter_map(|line| line.strip_prefix("GIT_COMMIT=\""))
+        .filter_map(|commit| commit.strip_suffix('"'))
+        .next()
+    {
+        if !commit.is_empty() {
+            return commit.to_string();
+        }
+    }
+
+    eprintln!("error: could not find commit hash for downloading libgccjit");
+    panic!();
+}