Add support for using cg_clif to bootstrap rustc

Author: bjorn3

Cargo.toml

@@ -31,6 +31,7 @@ members = [
 ]
 exclude = [
   "build",
+  "compiler/rustc_codegen_cranelift",
   # HACK(eddyb) This hardcodes the fact that our CI uses `/checkout/obj`.
   "obj",
 ]

compiler/rustc_driver/src/lib.rs

@@ -22,7 +22,7 @@ use rustc_errors::registry::{InvalidErrorCode, Registry};
 use rustc_errors::{ErrorReported, PResult};
 use rustc_feature::{find_gated_cfg, UnstableFeatures};
 use rustc_hir::def_id::LOCAL_CRATE;
-use rustc_interface::util::{collect_crate_types, get_builtin_codegen_backend};
+use rustc_interface::util::{self, collect_crate_types, get_builtin_codegen_backend};
 use rustc_interface::{interface, Queries};
 use rustc_lint::LintStore;
 use rustc_metadata::locator;
@@ -793,37 +793,24 @@ impl RustcDefaultCalls {
     }
 }
 
-/// Returns a version string such as "0.12.0-dev".
-fn release_str() -> Option<&'static str> {
-    option_env!("CFG_RELEASE")
-}
-
-/// Returns the full SHA1 hash of HEAD of the Git repo from which rustc was built.
-fn commit_hash_str() -> Option<&'static str> {
-    option_env!("CFG_VER_HASH")
-}
-
-/// Returns the "commit date" of HEAD of the Git repo from which rustc was built as a static string.
-fn commit_date_str() -> Option<&'static str> {
-    option_env!("CFG_VER_DATE")
-}
-
 /// Prints version information
 pub fn version(binary: &str, matches: &getopts::Matches) {
     let verbose = matches.opt_present("verbose");
 
-    println!("{} {}", binary, option_env!("CFG_VERSION").unwrap_or("unknown version"));
+    println!("{} {}", binary, util::version_str().unwrap_or("unknown version"));
 
     if verbose {
         fn unw(x: Option<&str>) -> &str {
             x.unwrap_or("unknown")
         }
         println!("binary: {}", binary);
-        println!("commit-hash: {}", unw(commit_hash_str()));
-        println!("commit-date: {}", unw(commit_date_str()));
+        println!("commit-hash: {}", unw(util::commit_hash_str()));
+        println!("commit-date: {}", unw(util::commit_date_str()));
         println!("host: {}", config::host_triple());
-        println!("release: {}", unw(release_str()));
-        get_builtin_codegen_backend("llvm")().print_version();
+        println!("release: {}", unw(util::release_str()));
+        if cfg!(llvm) {
+            get_builtin_codegen_backend("llvm")().print_version();
+        }
     }
 }
 
@@ -1109,7 +1096,9 @@ pub fn handle_options(args: &[String]) -> Option<getopts::Matches> {
     }
 
     if cg_flags.iter().any(|x| *x == "passes=list") {
-        get_builtin_codegen_backend("llvm")().print_passes();
+        if cfg!(llvm) {
+            get_builtin_codegen_backend("llvm")().print_passes();
+        }
         return None;
     }
 
@@ -1237,7 +1226,7 @@ pub fn report_ice(info: &panic::PanicInfo<'_>, bug_report_url: &str) {
         format!("we would appreciate a bug report: {}", bug_report_url).into(),
         format!(
             "rustc {} running on {}",
-            option_env!("CFG_VERSION").unwrap_or("unknown_version"),
+            util::version_str().unwrap_or("unknown_version"),
             config::host_triple()
         )
         .into(),

compiler/rustc_interface/src/util.rs

@@ -24,11 +24,13 @@ use rustc_span::source_map::FileLoader;
 use rustc_span::symbol::{sym, Symbol};
 use smallvec::SmallVec;
 use std::env;
+use std::env::consts::{DLL_PREFIX, DLL_SUFFIX};
 use std::io::{self, Write};
 use std::lazy::SyncOnceCell;
 use std::mem;
 use std::ops::DerefMut;
 use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex, Once};
 #[cfg(not(parallel_compiler))]
 use std::{panic, thread};
@@ -238,7 +240,19 @@ pub fn get_codegen_backend(sopts: &config::Options) -> Box<dyn CodegenBackend> {
     static mut LOAD: fn() -> Box<dyn CodegenBackend> = || unreachable!();
 
     INIT.call_once(|| {
-        let codegen_name = sopts.debugging_opts.codegen_backend.as_deref().unwrap_or("llvm");
+        #[cfg(feature = "llvm")]
+        const DEFAULT_CODEGEN_BACKEND: &'static str = "llvm";
+
+        #[cfg(not(feature = "llvm"))]
+        const DEFAULT_CODEGEN_BACKEND: &'static str = "cranelift";
+
+        let codegen_name = sopts
+            .debugging_opts
+            .codegen_backend
+            .as_ref()
+            .map(|name| &name[..])
+            .unwrap_or(DEFAULT_CODEGEN_BACKEND);
+
         let backend = match codegen_name {
             filename if filename.contains('.') => load_backend_from_dylib(filename.as_ref()),
             codegen_name => get_builtin_codegen_backend(codegen_name),
@@ -367,15 +381,102 @@ fn sysroot_candidates() -> Vec<PathBuf> {
 }
 
 pub fn get_builtin_codegen_backend(backend_name: &str) -> fn() -> Box<dyn CodegenBackend> {
-    #[cfg(feature = "llvm")]
-    {
-        if backend_name == "llvm" {
-            return rustc_codegen_llvm::LlvmCodegenBackend::new;
+    match backend_name {
+        #[cfg(feature = "llvm")]
+        "llvm" => rustc_codegen_llvm::LlvmCodegenBackend::new,
+        _ => get_codegen_sysroot(backend_name),
+    }
+}
+
+pub fn get_codegen_sysroot(backend_name: &str) -> fn() -> Box<dyn CodegenBackend> {
+    // For now we only allow this function to be called once as it'll dlopen a
+    // few things, which seems to work best if we only do that once. In
+    // general this assertion never trips due to the once guard in `get_codegen_backend`,
+    // but there's a few manual calls to this function in this file we protect
+    // against.
+    static LOADED: AtomicBool = AtomicBool::new(false);
+    assert!(
+        !LOADED.fetch_or(true, Ordering::SeqCst),
+        "cannot load the default codegen backend twice"
+    );
+
+    let target = session::config::host_triple();
+    let sysroot_candidates = sysroot_candidates();
+
+    let sysroot = sysroot_candidates
+        .iter()
+        .map(|sysroot| {
+            let libdir = filesearch::relative_target_lib_path(&sysroot, &target);
+            sysroot.join(libdir).with_file_name("codegen-backends")
+        })
+        .filter(|f| {
+            info!("codegen backend candidate: {}", f.display());
+            f.exists()
+        })
+        .next();
+    let sysroot = sysroot.unwrap_or_else(|| {
+        let candidates = sysroot_candidates
+            .iter()
+            .map(|p| p.display().to_string())
+            .collect::<Vec<_>>()
+            .join("\n* ");
+        let err = format!(
+            "failed to find a `codegen-backends` folder \
+                           in the sysroot candidates:\n* {}",
+            candidates
+        );
+        early_error(ErrorOutputType::default(), &err);
+    });
+    info!("probing {} for a codegen backend", sysroot.display());
+
+    let d = sysroot.read_dir().unwrap_or_else(|e| {
+        let err = format!(
+            "failed to load default codegen backend, couldn't \
+                           read `{}`: {}",
+            sysroot.display(),
+            e
+        );
+        early_error(ErrorOutputType::default(), &err);
+    });
+
+    let mut file: Option<PathBuf> = None;
+
+    let expected_name =
+        format!("rustc_codegen_{}-{}", backend_name, release_str().expect("CFG_RELEASE"));
+    for entry in d.filter_map(|e| e.ok()) {
+        let path = entry.path();
+        let filename = match path.file_name().and_then(|s| s.to_str()) {
+            Some(s) => s,
+            None => continue,
+        };
+        if !(filename.starts_with(DLL_PREFIX) && filename.ends_with(DLL_SUFFIX)) {
+            continue;
         }
+        let name = &filename[DLL_PREFIX.len()..filename.len() - DLL_SUFFIX.len()];
+        if name != expected_name {
+            continue;
+        }
+        if let Some(ref prev) = file {
+            let err = format!(
+                "duplicate codegen backends found\n\
+                               first:  {}\n\
+                               second: {}\n\
+            ",
+                prev.display(),
+                path.display()
+            );
+            early_error(ErrorOutputType::default(), &err);
+        }
+        file = Some(path.clone());
     }
 
-    let err = format!("unsupported builtin codegen backend `{}`", backend_name);
-    early_error(ErrorOutputType::default(), &err);
+    match file {
+        Some(ref s) => load_backend_from_dylib(s),
+        None => {
+            let err = format!("unsupported builtin codegen backend `{}`", backend_name);
+            early_error(ErrorOutputType::default(), &err);
+        }
+    }
 }
 
 pub(crate) fn compute_crate_disambiguator(session: &Session) -> CrateDisambiguator {
@@ -782,3 +883,23 @@ impl<'a> MutVisitor for ReplaceBodyWithLoop<'a, '_> {
         noop_visit_mac(mac, self)
     }
 }
+
+/// Returns a version string such as "rustc 1.46.0 (04488afe3 2020-08-24)"
+pub fn version_str() -> Option<&'static str> {
+    option_env!("CFG_VERSION")
+}
+
+/// Returns a version string such as "0.12.0-dev".
+pub fn release_str() -> Option<&'static str> {
+    option_env!("CFG_RELEASE")
+}
+
+/// Returns the full SHA1 hash of HEAD of the Git repo from which rustc was built.
+pub fn commit_hash_str() -> Option<&'static str> {
+    option_env!("CFG_VER_HASH")
+}
+
+/// Returns the "commit date" of HEAD of the Git repo from which rustc was built as a static string.
+pub fn commit_date_str() -> Option<&'static str> {
+    option_env!("CFG_VER_DATE")
+}

config.toml.example

@@ -478,7 +478,7 @@ changelog-seen = 2
 
 # This is an array of the codegen backends that will be compiled for the rustc
 # that's being compiled. The default is to only build the LLVM codegen backend,
-# and currently the only standard option supported is `"llvm"`
+# and currently the only standard options supported are `"llvm"` and `"cranelift"`.
 #codegen-backends = ["llvm"]
 
 # Indicates whether LLD will be compiled and made available in the sysroot for

rustfmt.toml

@@ -16,6 +16,7 @@ ignore = [
     # do not format submodules
     "library/backtrace",
     "library/stdarch",
+    "compiler/rustc_codegen_cranelift",
     "src/doc/book",
     "src/doc/edition-guide",
     "src/doc/embedded-book",

src/bootstrap/bootstrap.py

@@ -962,8 +962,12 @@ def ensure_vendored(self):
         # the rust git repository is updated. Normal development usually does
         # not use vendoring, so hopefully this isn't too much of a problem.
         if self.use_vendored_sources and not os.path.exists(vendor_dir):
-            run([self.cargo(), "vendor", "--sync=./src/tools/rust-analyzer/Cargo.toml"],
-                verbose=self.verbose, cwd=self.rust_root)
+            run([
+                self.cargo(),
+                "vendor",
+                "--sync=./src/tools/rust-analyzer/Cargo.toml",
+                "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
+            ], verbose=self.verbose, cwd=self.rust_root)
 
 
 def bootstrap(help_triggered):

src/bootstrap/builder.rs

@@ -344,6 +344,7 @@ impl<'a> Builder<'a> {
             Kind::Build => describe!(
                 compile::Std,
                 compile::Rustc,
+                compile::CodegenBackend,
                 compile::StartupObjects,
                 tool::BuildManifest,
                 tool::Rustbook,
@@ -370,9 +371,14 @@ impl<'a> Builder<'a> {
                 tool::CargoMiri,
                 native::Lld
             ),
-            Kind::Check | Kind::Clippy | Kind::Fix | Kind::Format => {
-                describe!(check::Std, check::Rustc, check::Rustdoc, check::Clippy, check::Bootstrap)
-            }
+            Kind::Check | Kind::Clippy | Kind::Fix | Kind::Format => describe!(
+                check::Std,
+                check::Rustc,
+                check::Rustdoc,
+                check::CodegenBackend,
+                check::Clippy,
+                check::Bootstrap
+            ),
             Kind::Test => describe!(
                 crate::toolstate::ToolStateCheck,
                 test::ExpandYamlAnchors,
@@ -630,6 +636,10 @@ impl<'a> Builder<'a> {
         self.ensure(Libdir { compiler, target })
     }
 
+    pub fn sysroot_codegen_backends(&self, compiler: Compiler) -> PathBuf {
+        self.sysroot_libdir(compiler, compiler.host).with_file_name("codegen-backends")
+    }
+
     /// Returns the compiler's libdir where it stores the dynamic libraries that
     /// it itself links against.
     ///
@@ -698,6 +708,15 @@ impl<'a> Builder<'a> {
         }
     }
 
+    /// Gets the paths to all of the compiler's codegen backends.
+    fn codegen_backends(&self, compiler: Compiler) -> impl Iterator<Item = PathBuf> {
+        fs::read_dir(self.sysroot_codegen_backends(compiler))
+            .into_iter()
+            .flatten()
+            .filter_map(Result::ok)
+            .map(|entry| entry.path())
+    }
+
     pub fn rustdoc(&self, compiler: Compiler) -> PathBuf {
         self.ensure(tool::Rustdoc { compiler })
     }
@@ -762,6 +781,12 @@ impl<'a> Builder<'a> {
         let mut cargo = Command::new(&self.initial_cargo);
         let out_dir = self.stage_out(compiler, mode);
 
+        // Codegen backends are not yet tracked by -Zbinary-dep-depinfo,
+        // so we need to explicitly clear out if they've been updated.
+        for backend in self.codegen_backends(compiler) {
+            self.clear_if_dirty(&out_dir, &backend);
+        }
+
         if cmd == "doc" || cmd == "rustdoc" {
             let my_out = match mode {
                 // This is the intended out directory for compiler documentation.
@@ -843,7 +868,7 @@ impl<'a> Builder<'a> {
 
         match mode {
             Mode::Std | Mode::ToolBootstrap | Mode::ToolStd => {}
-            Mode::Rustc | Mode::ToolRustc => {
+            Mode::Rustc | Mode::Codegen | Mode::ToolRustc => {
                 // Build proc macros both for the host and the target
                 if target != compiler.host && cmd != "check" {
                     cargo.arg("-Zdual-proc-macros");
@@ -904,6 +929,8 @@ impl<'a> Builder<'a> {
             // problem, somehow -- not really clear why -- but we know that this
             // fixes things.
             Mode::ToolRustc => metadata.push_str("tool-rustc"),
+            // Same for codegen backends.
+            Mode::Codegen => metadata.push_str("codegen"),
             _ => {}
         }
         cargo.env("__CARGO_DEFAULT_LIB_METADATA", &metadata);
@@ -1030,7 +1057,7 @@ impl<'a> Builder<'a> {
         }
 
         let debuginfo_level = match mode {
-            Mode::Rustc => self.config.rust_debuginfo_level_rustc,
+            Mode::Rustc | Mode::Codegen => self.config.rust_debuginfo_level_rustc,
             Mode::Std => self.config.rust_debuginfo_level_std,
             Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolRustc => {
                 self.config.rust_debuginfo_level_tools