Auto merge of rust-lang#96687 - jyn514:download-rustc, r=Mark-Simulacrum

Move download-rustc from python to rustbuild

- Remove download-rustc handling from bootstrap.py
- Allow a custom `pattern` in `builder.unpack()`
- Only download rustc once another part of bootstrap depends on it.

  This is somewhat necessary since the download functions rely on having a full
  `Builder`, which isn't available until after config parsing finishes.

Helps with rust-lang#94829.

Author: bors

src/bootstrap/bootstrap.py

@@ -444,9 +444,8 @@ def __init__(self):
         self.verbose = False
         self.git_version = None
         self.nix_deps_dir = None
-        self.rustc_commit = None
 
-    def download_toolchain(self, stage0=True, rustc_channel=None):
+    def download_toolchain(self):
         """Fetch the build system for Rust, written in Rust
 
         This method will build a cache directory, then it will fetch the
@@ -456,45 +455,35 @@ def download_toolchain(self, stage0=True, rustc_channel=None):
         Each downloaded tarball is extracted, after that, the script
         will move all the content to the right place.
         """
-        if rustc_channel is None:
-            rustc_channel = self.stage0_compiler.version
-        bin_root = self.bin_root(stage0)
+        rustc_channel = self.stage0_compiler.version
+        bin_root = self.bin_root()
 
         key = self.stage0_compiler.date
-        if not stage0:
-            key += str(self.rustc_commit)
-        if self.rustc(stage0).startswith(bin_root) and \
-                (not os.path.exists(self.rustc(stage0)) or
-                 self.program_out_of_date(self.rustc_stamp(stage0), key)):
+        if self.rustc().startswith(bin_root) and \
+                (not os.path.exists(self.rustc()) or
+                 self.program_out_of_date(self.rustc_stamp(), key)):
             if os.path.exists(bin_root):
                 shutil.rmtree(bin_root)
             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
             filename = "rust-std-{}-{}{}".format(
                 rustc_channel, self.build, tarball_suffix)
             pattern = "rust-std-{}".format(self.build)
-            self._download_component_helper(filename, pattern, tarball_suffix, stage0)
+            self._download_component_helper(filename, pattern, tarball_suffix)
             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
                                               tarball_suffix)
-            self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
-            # download-rustc doesn't need its own cargo, it can just use beta's.
-            if stage0:
-                filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
-                                                tarball_suffix)
-                self._download_component_helper(filename, "cargo", tarball_suffix)
-                self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
-            else:
-                filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
-                self._download_component_helper(
-                    filename, "rustc-dev", tarball_suffix, stage0
-                )
+            self._download_component_helper(filename, "rustc", tarball_suffix)
+            filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
+                                            tarball_suffix)
+            self._download_component_helper(filename, "cargo", tarball_suffix)
+            self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
 
             self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
             self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
             lib_dir = "{}/lib".format(bin_root)
             for lib in os.listdir(lib_dir):
                 if lib.endswith(".so"):
                     self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
-            with output(self.rustc_stamp(stage0)) as rust_stamp:
+            with output(self.rustc_stamp()) as rust_stamp:
                 rust_stamp.write(key)
 
         if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
@@ -518,24 +507,17 @@ def download_toolchain(self, stage0=True, rustc_channel=None):
                     rustfmt_stamp.write(self.stage0_rustfmt.channel())
 
     def _download_component_helper(
-        self, filename, pattern, tarball_suffix, stage0=True, key=None
+        self, filename, pattern, tarball_suffix, key=None
     ):
         if key is None:
-            if stage0:
-                key = self.stage0_compiler.date
-            else:
-                key = self.rustc_commit
+            key = self.stage0_compiler.date
         cache_dst = os.path.join(self.build_dir, "cache")
         rustc_cache = os.path.join(cache_dst, key)
         if not os.path.exists(rustc_cache):
             os.makedirs(rustc_cache)
 
-        if stage0:
-            base = self._download_url
-            url = "dist/{}".format(key)
-        else:
-            base = "https://ci-artifacts.rust-lang.org"
-            url = "rustc-builds/{}".format(self.rustc_commit)
+        base = self._download_url
+        url = "dist/{}".format(key)
         tarball = os.path.join(rustc_cache, filename)
         if not os.path.exists(tarball):
             get(
@@ -544,9 +526,9 @@ def _download_component_helper(
                 tarball,
                 self.checksums_sha256,
                 verbose=self.verbose,
-                do_verify=stage0,
+                do_verify=True,
             )
-        unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
+        unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 
     def fix_bin_or_dylib(self, fname):
         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
@@ -644,62 +626,15 @@ def fix_bin_or_dylib(self, fname):
             print("warning: failed to call patchelf:", reason)
             return
 
-    # If `download-rustc` is set, download the most recent commit with CI artifacts
-    def maybe_download_ci_toolchain(self):
-        # If `download-rustc` is not set, default to rebuilding.
-        download_rustc = self.get_toml("download-rustc", section="rust")
-        if download_rustc is None or download_rustc == "false":
-            return None
-        assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
-
-        # Handle running from a directory other than the top level
-        rev_parse = ["git", "rev-parse", "--show-toplevel"]
-        top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
-        compiler = "{}/compiler/".format(top_level)
-        library = "{}/library/".format(top_level)
-
-        # Look for a version to compare to based on the current commit.
-        # Only commits merged by bors will have CI artifacts.
-        merge_base = [
-            "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
-            "--first-parent", "HEAD"
-        ]
-        commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
-        if not commit:
-            print("error: could not find commit hash for downloading rustc")
-            print("help: maybe your repository history is too shallow?")
-            print("help: consider disabling `download-rustc`")
-            print("help: or fetch enough history to include one upstream commit")
-            exit(1)
-
-        # Warn if there were changes to the compiler or standard library since the ancestor commit.
-        status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
-        if status != 0:
-            if download_rustc == "if-unchanged":
-                if self.verbose:
-                    print("warning: saw changes to compiler/ or library/ since {}; " \
-                          "ignoring `download-rustc`".format(commit))
-                return None
-            print("warning: `download-rustc` is enabled, but there are changes to " \
-                  "compiler/ or library/")
-
-        if self.verbose:
-            print("using downloaded stage2 artifacts from CI (commit {})".format(commit))
-        self.rustc_commit = commit
-        # FIXME: support downloading artifacts from the beta channel
-        self.download_toolchain(False, "nightly")
-
-    def rustc_stamp(self, stage0):
+    def rustc_stamp(self):
         """Return the path for .rustc-stamp at the given stage
 
         >>> rb = RustBuild()
         >>> rb.build_dir = "build"
-        >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
-        True
-        >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
+        >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
         True
         """
-        return os.path.join(self.bin_root(stage0), '.rustc-stamp')
+        return os.path.join(self.bin_root(), '.rustc-stamp')
 
     def rustfmt_stamp(self):
         """Return the path for .rustfmt-stamp
@@ -709,7 +644,7 @@ def rustfmt_stamp(self):
         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
         True
         """
-        return os.path.join(self.bin_root(True), '.rustfmt-stamp')
+        return os.path.join(self.bin_root(), '.rustfmt-stamp')
 
     def program_out_of_date(self, stamp_path, key):
         """Check if the given program stamp is out of date"""
@@ -718,26 +653,21 @@ def program_out_of_date(self, stamp_path, key):
         with open(stamp_path, 'r') as stamp:
             return key != stamp.read()
 
-    def bin_root(self, stage0):
+    def bin_root(self):
         """Return the binary root directory for the given stage
 
         >>> rb = RustBuild()
         >>> rb.build_dir = "build"
-        >>> rb.bin_root(True) == os.path.join("build", "stage0")
-        True
-        >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
+        >>> rb.bin_root() == os.path.join("build", "stage0")
         True
 
         When the 'build' property is given should be a nested directory:
 
         >>> rb.build = "devel"
-        >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
+        >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
         True
         """
-        if stage0:
-            subdir = "stage0"
-        else:
-            subdir = "ci-rustc"
+        subdir = "stage0"
         return os.path.join(self.build_dir, self.build, subdir)
 
     def get_toml(self, key, section=None):
@@ -785,37 +715,33 @@ def cargo(self):
         """Return config path for cargo"""
         return self.program_config('cargo')
 
-    def rustc(self, stage0):
+    def rustc(self):
         """Return config path for rustc"""
-        return self.program_config('rustc', stage0)
+        return self.program_config('rustc')
 
     def rustfmt(self):
         """Return config path for rustfmt"""
         if self.stage0_rustfmt is None:
             return None
         return self.program_config('rustfmt')
 
-    def program_config(self, program, stage0=True):
+    def program_config(self, program):
         """Return config path for the given program at the given stage
 
         >>> rb = RustBuild()
         >>> rb.config_toml = 'rustc = "rustc"\\n'
         >>> rb.program_config('rustc')
         'rustc'
         >>> rb.config_toml = ''
-        >>> cargo_path = rb.program_config('cargo', True)
-        >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
-        ... "bin", "cargo")
-        True
-        >>> cargo_path = rb.program_config('cargo', False)
-        >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
+        >>> cargo_path = rb.program_config('cargo')
+        >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
         ... "bin", "cargo")
         True
         """
         config = self.get_toml(program)
         if config:
             return os.path.expanduser(config)
-        return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
+        return os.path.join(self.bin_root(), "bin", "{}{}".format(
             program, self.exe_suffix()))
 
     @staticmethod
@@ -871,14 +797,14 @@ def build_bootstrap(self):
         if "CARGO_BUILD_TARGET" in env:
             del env["CARGO_BUILD_TARGET"]
         env["CARGO_TARGET_DIR"] = build_dir
-        env["RUSTC"] = self.rustc(True)
-        env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
+        env["RUSTC"] = self.rustc()
+        env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
             (os.pathsep + env["LD_LIBRARY_PATH"]) \
             if "LD_LIBRARY_PATH" in env else ""
-        env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
+        env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
             if "DYLD_LIBRARY_PATH" in env else ""
-        env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
+        env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
             (os.pathsep + env["LIBRARY_PATH"]) \
             if "LIBRARY_PATH" in env else ""
 
@@ -900,7 +826,7 @@ def build_bootstrap(self):
         if self.get_toml("deny-warnings", "rust") != "false":
             env["RUSTFLAGS"] += " -Dwarnings"
 
-        env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
+        env["PATH"] = os.path.join(self.bin_root(), "bin") + \
             os.pathsep + env["PATH"]
         if not os.path.isfile(self.cargo()):
             raise Exception("no cargo executable found at `{}`".format(
@@ -1171,8 +1097,6 @@ def bootstrap(help_triggered):
 
     # Fetch/build the bootstrap
     build.download_toolchain()
-    # Download the master compiler if `download-rustc` is set
-    build.maybe_download_ci_toolchain()
     sys.stdout.flush()
     build.ensure_vendored()
     build.build_bootstrap()
@@ -1184,8 +1108,6 @@ def bootstrap(help_triggered):
     env = os.environ.copy()
     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
     env["BOOTSTRAP_PYTHON"] = sys.executable
-    if build.rustc_commit is not None:
-        env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
     run(args, env=env, verbose=build.verbose, is_bootstrap=True)