Auto merge of rust-lang#95170 - jyn514:ci-llvm, r=Mark-Simulacrum

Move `download-ci-llvm` out of bootstrap.py

This is ready for review. It has been tested on Windows, Linux, and NixOS.

The second commit ports the changes from rust-lang#95234 to Rust; I can remove it if desired.

Helps with rust-lang#94829.

As a follow-up, this makes it possible to avoid downloading llvm until it's needed for building `rustc_llvm`; it would be nice to do that, but it shouldn't go in the first draft. It might also be possible to avoid requiring python until tests run (currently there's a check in `sanity.rs`), but I haven't looked too much into that.

`@rustbot` label +A-rustbuild

Author: bors

Cargo.lock

@@ -225,8 +225,10 @@ dependencies = [
  "pretty_assertions",
  "serde",
  "serde_json",
+ "tar",
  "toml",
  "winapi",
+ "xz2",
 ]
 
 [[package]]

src/bootstrap/Cargo.toml

@@ -42,10 +42,12 @@ cc = "1.0.69"
 libc = "0.2"
 serde = { version = "1.0.8", features = ["derive"] }
 serde_json = "1.0.2"
+tar = "0.4"
 toml = "0.5"
 ignore = "0.4.10"
 opener = "0.5"
 once_cell = "1.7.2"
+xz2 = "0.1"
 
 [target.'cfg(windows)'.dependencies.winapi]
 version = "0.3"

src/bootstrap/bootstrap.py

@@ -500,81 +500,6 @@ def download_toolchain(self, stage0=True, rustc_channel=None):
                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
                     rustfmt_stamp.write(self.stage0_rustfmt.channel())
 
-        # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
-        if self.downloading_llvm() and stage0:
-            # We want the most recent LLVM submodule update to avoid downloading
-            # LLVM more often than necessary.
-            #
-            # This git command finds that commit SHA, looking for bors-authored
-            # commits that modified src/llvm-project or other relevant version
-            # stamp files.
-            #
-            # This works even in a repository that has not yet initialized
-            # submodules.
-            top_level = subprocess.check_output([
-                "git", "rev-parse", "--show-toplevel",
-            ]).decode(sys.getdefaultencoding()).strip()
-            llvm_sha = subprocess.check_output([
-                "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
-                "--first-parent", "HEAD",
-                "--",
-                "{}/src/llvm-project".format(top_level),
-                "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
-                # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
-                "{}/src/version".format(top_level)
-            ]).decode(sys.getdefaultencoding()).strip()
-            llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
-            llvm_root = self.llvm_root()
-            llvm_lib = os.path.join(llvm_root, "lib")
-            if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
-                self._download_ci_llvm(llvm_sha, llvm_assertions)
-                for binary in ["llvm-config", "FileCheck"]:
-                    self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
-                for lib in os.listdir(llvm_lib):
-                    if lib.endswith(".so"):
-                        self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
-                with output(self.llvm_stamp()) as llvm_stamp:
-                    llvm_stamp.write(llvm_sha + str(llvm_assertions))
-
-    def downloading_llvm(self):
-        opt = self.get_toml('download-ci-llvm', 'llvm')
-        # This is currently all tier 1 targets and tier 2 targets with host tools
-        # (since others may not have CI artifacts)
-        # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
-        supported_platforms = [
-            # tier 1
-            "aarch64-unknown-linux-gnu",
-            "i686-pc-windows-gnu",
-            "i686-pc-windows-msvc",
-            "i686-unknown-linux-gnu",
-            "x86_64-unknown-linux-gnu",
-            "x86_64-apple-darwin",
-            "x86_64-pc-windows-gnu",
-            "x86_64-pc-windows-msvc",
-            # tier 2 with host tools
-            "aarch64-apple-darwin",
-            "aarch64-pc-windows-msvc",
-            "aarch64-unknown-linux-musl",
-            "arm-unknown-linux-gnueabi",
-            "arm-unknown-linux-gnueabihf",
-            "armv7-unknown-linux-gnueabihf",
-            "mips-unknown-linux-gnu",
-            "mips64-unknown-linux-gnuabi64",
-            "mips64el-unknown-linux-gnuabi64",
-            "mipsel-unknown-linux-gnu",
-            "powerpc-unknown-linux-gnu",
-            "powerpc64-unknown-linux-gnu",
-            "powerpc64le-unknown-linux-gnu",
-            "riscv64gc-unknown-linux-gnu",
-            "s390x-unknown-linux-gnu",
-            "x86_64-unknown-freebsd",
-            "x86_64-unknown-illumos",
-            "x86_64-unknown-linux-musl",
-            "x86_64-unknown-netbsd",
-        ]
-        return opt == "true" \
-            or (opt == "if-available" and self.build in supported_platforms)
-
     def _download_component_helper(
         self, filename, pattern, tarball_suffix, stage0=True, key=None
     ):
@@ -606,53 +531,6 @@ def _download_component_helper(
             )
         unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
 
-    def _download_ci_llvm(self, llvm_sha, llvm_assertions):
-        if not llvm_sha:
-            print("error: could not find commit hash for downloading LLVM")
-            print("help: maybe your repository history is too shallow?")
-            print("help: consider disabling `download-ci-llvm`")
-            print("help: or fetch enough history to include one upstream commit")
-            exit(1)
-        cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
-        cache_dst = os.path.join(self.build_dir, "cache")
-        rustc_cache = os.path.join(cache_dst, cache_prefix)
-        if not os.path.exists(rustc_cache):
-            os.makedirs(rustc_cache)
-
-        base = "https://ci-artifacts.rust-lang.org"
-        url = "rustc-builds/{}".format(llvm_sha)
-        if llvm_assertions:
-            url = url.replace('rustc-builds', 'rustc-builds-alt')
-        # ci-artifacts are only stored as .xz, not .gz
-        if not support_xz():
-            print("error: XZ support is required to download LLVM")
-            print("help: consider disabling `download-ci-llvm` or using python3")
-            exit(1)
-        tarball_suffix = '.tar.xz'
-        filename = "rust-dev-nightly-" + self.build + tarball_suffix
-        tarball = os.path.join(rustc_cache, filename)
-        if not os.path.exists(tarball):
-            help_on_error = "error: failed to download llvm from ci"
-            help_on_error += "\nhelp: old builds get deleted after a certain time"
-            help_on_error += "\nhelp: if trying to compile an old commit of rustc,"
-            help_on_error += " disable `download-ci-llvm` in config.toml:"
-            help_on_error += "\n"
-            help_on_error += "\n[llvm]"
-            help_on_error += "\ndownload-ci-llvm = false"
-            help_on_error += "\n"
-            get(
-                base,
-                "{}/{}".format(url, filename),
-                tarball,
-                self.checksums_sha256,
-                verbose=self.verbose,
-                do_verify=False,
-                help_on_error=help_on_error,
-            )
-        unpack(tarball, tarball_suffix, self.llvm_root(),
-                match="rust-dev",
-                verbose=self.verbose)
-
     def fix_bin_or_dylib(self, fname):
         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
         or the RPATH section, to fix the dynamic library search path
@@ -816,17 +694,6 @@ def rustfmt_stamp(self):
         """
         return os.path.join(self.bin_root(True), '.rustfmt-stamp')
 
-    def llvm_stamp(self):
-        """Return the path for .llvm-stamp
-
-        >>> rb = RustBuild()
-        >>> rb.build_dir = "build"
-        >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
-        True
-        """
-        return os.path.join(self.llvm_root(), '.llvm-stamp')
-
-
     def program_out_of_date(self, stamp_path, key):
         """Check if the given program stamp is out of date"""
         if not os.path.exists(stamp_path) or self.clean:
@@ -856,22 +723,6 @@ def bin_root(self, stage0):
             subdir = "ci-rustc"
         return os.path.join(self.build_dir, self.build, subdir)
 
-    def llvm_root(self):
-        """Return the CI LLVM root directory
-
-        >>> rb = RustBuild()
-        >>> rb.build_dir = "build"
-        >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
-        True
-
-        When the 'build' property is given should be a nested directory:
-
-        >>> rb.build = "devel"
-        >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
-        True
-        """
-        return os.path.join(self.build_dir, self.build, "ci-llvm")
-
     def get_toml(self, key, section=None):
         """Returns the value of the given key in config.toml, otherwise returns None
 

src/bootstrap/builder.rs

@@ -12,7 +12,6 @@ use std::process::Command;
 use std::time::{Duration, Instant};
 
 use crate::cache::{Cache, Interned, INTERNER};
-use crate::check;
 use crate::compile;
 use crate::config::{SplitDebuginfo, TargetSelection};
 use crate::dist;
@@ -25,6 +24,7 @@ use crate::test;
 use crate::tool::{self, SourceType};
 use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t};
 use crate::EXTRA_CHECK_CFGS;
+use crate::{check, Config};
 use crate::{Build, CLang, DocTests, GitRepo, Mode};
 
 pub use crate::Compiler;
@@ -960,6 +960,11 @@ impl<'a> Builder<'a> {
         None
     }
 
+    /// Convenience wrapper to allow `builder.llvm_link_shared()` instead of `builder.config.llvm_link_shared(&builder)`.
+    pub(crate) fn llvm_link_shared(&self) -> bool {
+        Config::llvm_link_shared(self)
+    }
+
     /// Prepares an invocation of `cargo` to be run.
     ///
     /// This will create a `Command` that represents a pending execution of

src/bootstrap/compile.rs

@@ -737,7 +737,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS
             );
             cargo.env("LLVM_STATIC_STDCPP", file);
         }
-        if builder.config.llvm_link_shared {
+        if builder.llvm_link_shared() {
             cargo.env("LLVM_LINK_SHARED", "1");
         }
         if builder.config.llvm_use_libcxx {

src/bootstrap/config.rs

@@ -3,6 +3,7 @@
 //! This module implements parsing `config.toml` configuration files to tweak
 //! how the build runs.
 
+use std::cell::Cell;
 use std::cmp;
 use std::collections::{HashMap, HashSet};
 use std::env;
@@ -11,7 +12,7 @@ use std::fs;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
-use crate::builder::TaskPath;
+use crate::builder::{Builder, TaskPath};
 use crate::cache::{Interned, INTERNER};
 use crate::channel::GitInfo;
 pub use crate::flags::Subcommand;
@@ -69,13 +70,14 @@ pub struct Config {
     pub test_compare_mode: bool,
     pub llvm_libunwind: LlvmLibunwind,
     pub color: Color,
+    pub patch_binaries_for_nix: bool,
 
     pub on_fail: Option<String>,
     pub stage: u32,
     pub keep_stage: Vec<u32>,
     pub keep_stage_std: Vec<u32>,
     pub src: PathBuf,
-    // defaults to `config.toml`
+    /// defaults to `config.toml`
     pub config: PathBuf,
     pub jobs: Option<u32>,
     pub cmd: Subcommand,
@@ -96,7 +98,11 @@ pub struct Config {
     pub llvm_release_debuginfo: bool,
     pub llvm_version_check: bool,
     pub llvm_static_stdcpp: bool,
-    pub llvm_link_shared: bool,
+    /// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm.
+    #[cfg(not(test))]
+    llvm_link_shared: Cell<Option<bool>>,
+    #[cfg(test)]
+    pub llvm_link_shared: Cell<Option<bool>>,
     pub llvm_clang_cl: Option<String>,
     pub llvm_targets: Option<String>,
     pub llvm_experimental_targets: Option<String>,
@@ -858,6 +864,7 @@ impl Config {
         set(&mut config.local_rebuild, build.local_rebuild);
         set(&mut config.print_step_timings, build.print_step_timings);
         set(&mut config.print_step_rusage, build.print_step_rusage);
+        set(&mut config.patch_binaries_for_nix, build.patch_binaries_for_nix);
 
         config.verbose = cmp::max(config.verbose, flags.verbose);
 
@@ -913,7 +920,9 @@ impl Config {
             set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo);
             set(&mut config.llvm_version_check, llvm.version_check);
             set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp);
-            set(&mut config.llvm_link_shared, llvm.link_shared);
+            if let Some(v) = llvm.link_shared {
+                config.llvm_link_shared.set(Some(v));
+            }
             config.llvm_targets = llvm.targets.clone();
             config.llvm_experimental_targets = llvm.experimental_targets.clone();
             config.llvm_link_jobs = llvm.link_jobs;
@@ -983,6 +992,7 @@ impl Config {
                 check_ci_llvm!(llvm.optimize);
                 check_ci_llvm!(llvm.thin_lto);
                 check_ci_llvm!(llvm.release_debuginfo);
+                // CI-built LLVM can be either dynamic or static. We won't know until we download it.
                 check_ci_llvm!(llvm.link_shared);
                 check_ci_llvm!(llvm.static_libstdcpp);
                 check_ci_llvm!(llvm.targets);
@@ -1000,26 +1010,14 @@ impl Config {
                 check_ci_llvm!(llvm.clang);
                 check_ci_llvm!(llvm.build_config);
                 check_ci_llvm!(llvm.plugins);
-
-                // CI-built LLVM can be either dynamic or static.
-                let ci_llvm = config.out.join(&*config.build.triple).join("ci-llvm");
-                config.llvm_link_shared = if config.dry_run {
-                    // just assume dynamic for now
-                    true
-                } else {
-                    let link_type = t!(
-                        std::fs::read_to_string(ci_llvm.join("link-type.txt")),
-                        format!("CI llvm missing: {}", ci_llvm.display())
-                    );
-                    link_type == "dynamic"
-                };
             }
 
+            // NOTE: can never be hit when downloading from CI, since we call `check_ci_llvm!(thin_lto)` above.
             if config.llvm_thin_lto && llvm.link_shared.is_none() {
                 // If we're building with ThinLTO on, by default we want to link
                 // to LLVM shared, to avoid re-doing ThinLTO (which happens in
                 // the link step) with each stage.
-                config.llvm_link_shared = true;
+                config.llvm_link_shared.set(Some(true));
             }
         }
 
@@ -1274,6 +1272,42 @@ impl Config {
         }
     }
 
+    /// The absolute path to the downloaded LLVM artifacts.
+    pub(crate) fn ci_llvm_root(&self) -> PathBuf {
+        assert!(self.llvm_from_ci);
+        self.out.join(&*self.build.triple).join("ci-llvm")
+    }
+
+    /// Determine whether llvm should be linked dynamically.
+    ///
+    /// If `false`, llvm should be linked statically.
+    /// This is computed on demand since LLVM might have to first be downloaded from CI.
+    pub(crate) fn llvm_link_shared(builder: &Builder<'_>) -> bool {
+        let mut opt = builder.config.llvm_link_shared.get();
+        if opt.is_none() && builder.config.dry_run {
+            // just assume static for now - dynamic linking isn't supported on all platforms
+            return false;
+        }
+
+        let llvm_link_shared = *opt.get_or_insert_with(|| {
+            if builder.config.llvm_from_ci {
+                crate::native::maybe_download_ci_llvm(builder);
+                let ci_llvm = builder.config.ci_llvm_root();
+                let link_type = t!(
+                    std::fs::read_to_string(ci_llvm.join("link-type.txt")),
+                    format!("CI llvm missing: {}", ci_llvm.display())
+                );
+                link_type == "dynamic"
+            } else {
+                // unclear how thought-through this default is, but it maintains compatibility with
+                // previous behavior
+                false
+            }
+        });
+        builder.config.llvm_link_shared.set(opt);
+        llvm_link_shared
+    }
+
     pub fn verbose(&self) -> bool {
         self.verbose > 0
     }