Rollup merge of rust-lang#121976 - lu-zero:bootstrap-cache, r=onur-ozkan

Add an option to have an external download/bootstrap cache

Follow up from rust-lang#116697 to address rust-lang#116697 (review)

Author: matthiaskrgr

config.example.toml

@@ -300,6 +300,10 @@
 # This is only useful for verifying that rustc generates reproducible builds.
 #full-bootstrap = false
 
+# Set the bootstrap/download cache path. It is useful when building rust
+# repeatedly in a CI invironment.
+# bootstrap-cache-path = /shared/cache
+
 # Enable a build of the extended Rust tool set which is not only the compiler
 # but also tools such as Cargo. This will also produce "combined installers"
 # which are used to install Rust and Cargo together.

src/bootstrap/bootstrap.py

@@ -557,7 +557,9 @@ def download_toolchain(self):
                 shutil.rmtree(bin_root)
 
             key = self.stage0_compiler.date
-            cache_dst = os.path.join(self.build_dir, "cache")
+            cache_dst = (self.get_toml('bootstrap-cache-path', 'build') or
+                os.path.join(self.build_dir, "cache"))
+
             rustc_cache = os.path.join(cache_dst, key)
             if not os.path.exists(rustc_cache):
                 os.makedirs(rustc_cache)

src/bootstrap/configure.py

@@ -149,6 +149,7 @@ def v(*args):
 # (others are conditionally saved).
 o("manage-submodules", "build.submodules", "let the build manage the git submodules")
 o("full-bootstrap", "build.full-bootstrap", "build three compilers instead of two (not recommended except for testing reproducible builds)")
+o("bootstrap-cache-path", "build.bootstrap-cache-path", "use provided path for the bootstrap cache")
 o("extended", "build.extended", "build an extended rust tool set")
 
 v("tools", None, "List of extended tools will be installed")

src/bootstrap/src/core/config/config.rs

@@ -161,6 +161,7 @@ pub struct Config {
     pub vendor: bool,
     pub target_config: HashMap<TargetSelection, Target>,
     pub full_bootstrap: bool,
+    pub bootstrap_cache_path: Option<PathBuf>,
     pub extended: bool,
     pub tools: Option<HashSet<String>>,
     pub sanitizers: bool,
@@ -827,6 +828,7 @@ define_config! {
         locked_deps: Option<bool> = "locked-deps",
         vendor: Option<bool> = "vendor",
         full_bootstrap: Option<bool> = "full-bootstrap",
+        bootstrap_cache_path: Option<PathBuf> = "bootstrap-cache-path",
         extended: Option<bool> = "extended",
         tools: Option<HashSet<String>> = "tools",
         verbose: Option<usize> = "verbose",
@@ -1389,6 +1391,7 @@ impl Config {
             locked_deps,
             vendor,
             full_bootstrap,
+            bootstrap_cache_path,
             extended,
             tools,
             verbose,
@@ -1477,6 +1480,7 @@ impl Config {
         config.reuse = reuse.map(PathBuf::from);
         config.submodules = submodules;
         config.android_ndk = android_ndk;
+        config.bootstrap_cache_path = bootstrap_cache_path;
         set(&mut config.low_priority, low_priority);
         set(&mut config.compiler_docs, compiler_docs);
         set(&mut config.library_docs_private_items, library_docs_private_items);

src/bootstrap/src/core/download.rs

@@ -578,7 +578,9 @@ impl Config {
             return;
         }
 
-        let cache_dst = self.out.join("cache");
+        let cache_dst =
+            self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache"));
+
         let cache_dir = cache_dst.join(key);
         if !cache_dir.exists() {
             t!(fs::create_dir_all(&cache_dir));
@@ -705,7 +707,9 @@ download-rustc = false
         let llvm_assertions = self.llvm_assertions;
 
         let cache_prefix = format!("llvm-{llvm_sha}-{llvm_assertions}");
-        let cache_dst = self.out.join("cache");
+        let cache_dst =
+            self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache"));
+
         let rustc_cache = cache_dst.join(cache_prefix);
         if !rustc_cache.exists() {
             t!(fs::create_dir_all(&rustc_cache));

src/bootstrap/src/utils/change_tracker.rs

@@ -136,4 +136,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[
         severity: ChangeSeverity::Info,
         summary: "`x install` now skips providing tarball sources (under 'build/dist' path) to speed up the installation process.",
     },
+    ChangeInfo {
+        change_id: 121976,
+        severity: ChangeSeverity::Info,
+        summary: "A new `boostrap-cache-path` option has been introduced which can be utilized to modify the cache path for bootstrap.",
+    },
 ];