Auto merge of rust-lang#81601 - jyn514:llvm-on-demand, r=Mark-Simulacrum

Move llvm submodule updates to rustbuild

This enables better caching, since LLVM is only updated when needed, not
whenever x.py is run. Before, bootstrap.py had to use heuristics to
guess if LLVM would be needed, and updated the module more often than
necessary as a result.

This syncs the LLVM submodule only just before building the compiler, so
people working on the standard library never have to worry about it.
Example output:

```
Copying stage0 std from stage0 (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu / x86_64-unknown-linux-gnu)
Updating submodule src/llvm-project
Submodule 'src/llvm-project' (https://github.com/rust-lang/llvm-project.git) registered for path 'src/llvm-project'
Submodule path 'src/llvm-project': checked out 'f9a8d70b6e0365ac2172ca6b7f1de0341297458d'
```

Implements rust-lang#76653 (comment). This could be easily extended to other submodules, like `rust-by-example` and `rustc-dev-guide`, which aren't needed for cargo's workspace resolution.

Author: bors

src/bootstrap/bootstrap.py

@@ -991,28 +991,20 @@ def update_submodules(self):
         ).decode(default_encoding).splitlines()]
         filtered_submodules = []
         submodules_names = []
-        llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
-        external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
-        llvm_needed = not self.get_toml('codegen-backends', 'rust') \
-            or "llvm" in self.get_toml('codegen-backends', 'rust')
         for module in submodules:
+            # This is handled by native::Llvm in rustbuild, not here
             if module.endswith("llvm-project"):
-                # Don't sync the llvm-project submodule if an external LLVM was
-                # provided, if we are downloading LLVM or if the LLVM backend is
-                # not being built. Also, if the submodule has been initialized
-                # already, sync it anyways so that it doesn't mess up contributor
-                # pull requests.
-                if external_llvm_provided or not llvm_needed:
-                    if self.get_toml('lld') != 'true' and not llvm_checked_out:
-                        continue
+                continue
             check = self.check_submodule(module, slow_submodules)
             filtered_submodules.append((module, check))
             submodules_names.append(module)
         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
                                     cwd=self.rust_root, stdout=subprocess.PIPE)
         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
+        # { filename: hash }
         recorded_submodules = {}
         for data in recorded:
+            # [mode, kind, hash, filename]
             data = data.split()
             recorded_submodules[data[3]] = data[2]
         for module in filtered_submodules:

src/bootstrap/lib.rs

@@ -472,12 +472,22 @@ impl Build {
         slice::from_ref(&self.build.triple)
     }
 
+    /// If the LLVM submodule has been initialized already, sync it unconditionally. This avoids
+    /// contributors checking in a submodule change by accident.
+    pub fn maybe_update_llvm_submodule(&self) {
+        if self.in_tree_llvm_info.is_git() {
+            native::update_llvm_submodule(self);
+        }
+    }
+
     /// Executes the entire build, as configured by the flags and configuration.
     pub fn build(&mut self) {
         unsafe {
             job::setup(self);
         }
 
+        self.maybe_update_llvm_submodule();
+
         if let Subcommand::Format { check, paths } = &self.config.cmd {
             return format::format(self, *check, &paths);
         }

src/bootstrap/native.rs

@@ -21,7 +21,7 @@ use build_helper::{output, t};
 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
 use crate::config::TargetSelection;
 use crate::util::{self, exe};
-use crate::GitRepo;
+use crate::{Build, GitRepo};
 use build_helper::up_to_date;
 
 pub struct Meta {
@@ -91,6 +91,85 @@ pub fn prebuilt_llvm_config(
     Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() })
 }
 
+// modified from `check_submodule` and `update_submodule` in bootstrap.py
+pub(crate) fn update_llvm_submodule(build: &Build) {
+    let llvm_project = &Path::new("src").join("llvm-project");
+
+    fn dir_is_empty(dir: &Path) -> bool {
+        t!(std::fs::read_dir(dir)).next().is_none()
+    }
+
+    // NOTE: The check for the empty directory is here because when running x.py
+    // the first time, the llvm submodule won't be checked out. Check it out
+    // now so we can build it.
+    if !build.in_tree_llvm_info.is_git() && !dir_is_empty(&build.config.src.join(llvm_project)) {
+        return;
+    }
+
+    // check_submodule
+    let checked_out = if build.config.fast_submodules {
+        Some(output(
+            Command::new("git")
+                .args(&["rev-parse", "HEAD"])
+                .current_dir(build.config.src.join(llvm_project)),
+        ))
+    } else {
+        None
+    };
+
+    // update_submodules
+    let recorded = output(
+        Command::new("git")
+            .args(&["ls-tree", "HEAD"])
+            .arg(llvm_project)
+            .current_dir(&build.config.src),
+    );
+    let hash =
+        recorded.split(' ').nth(2).unwrap_or_else(|| panic!("unexpected output `{}`", recorded));
+
+    // update_submodule
+    if let Some(llvm_hash) = checked_out {
+        if hash == llvm_hash {
+            // already checked out
+            return;
+        }
+    }
+
+    println!("Updating submodule {}", llvm_project.display());
+    build.run(
+        Command::new("git")
+            .args(&["submodule", "-q", "sync"])
+            .arg(llvm_project)
+            .current_dir(&build.config.src),
+    );
+
+    // Try passing `--progress` to start, then run git again without if that fails.
+    let update = |progress: bool| {
+        let mut git = Command::new("git");
+        git.args(&["submodule", "update", "--init", "--recursive"]);
+        if progress {
+            git.arg("--progress");
+        }
+        git.arg(llvm_project).current_dir(&build.config.src);
+        git
+    };
+    // NOTE: doesn't use `try_run` because this shouldn't print an error if it fails.
+    if !update(true).status().map_or(false, |status| status.success()) {
+        build.run(&mut update(false));
+    }
+
+    build.run(
+        Command::new("git")
+            .args(&["reset", "-q", "--hard"])
+            .current_dir(build.config.src.join(llvm_project)),
+    );
+    build.run(
+        Command::new("git")
+            .args(&["clean", "-qdfx"])
+            .current_dir(build.config.src.join(llvm_project)),
+    );
+}
+
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Llvm {
     pub target: TargetSelection,
@@ -128,6 +207,9 @@ impl Step for Llvm {
                 Err(m) => m,
             };
 
+        if !builder.config.dry_run {
+            update_llvm_submodule(builder);
+        }
         if builder.config.llvm_link_shared
             && (target.contains("windows") || target.contains("apple-darwin"))
         {

src/build_helper/lib.rs

@@ -130,6 +130,7 @@ pub fn make(host: &str) -> PathBuf {
     }
 }
 
+#[track_caller]
 pub fn output(cmd: &mut Command) -> String {
     let output = match cmd.stderr(Stdio::inherit()).output() {
         Ok(status) => status,