Add checksum verification for rustfmt downloads

Author: jyn514

Cargo.lock

@@ -216,13 +216,15 @@ dependencies = [
  "cmake",
  "filetime",
  "getopts",
+ "hex 0.4.2",
  "ignore",
  "libc",
  "once_cell",
  "opener",
  "pretty_assertions 0.7.2",
  "serde",
  "serde_json",
+ "sha2",
  "sysinfo",
  "tar",
  "toml",

src/bootstrap/Cargo.toml

@@ -40,8 +40,10 @@ filetime = "0.2"
 getopts = "0.2.19"
 cc = "1.0.69"
 libc = "0.2"
+hex = "0.4"
 serde = { version = "1.0.8", features = ["derive"] }
 serde_json = "1.0.2"
+sha2 = "0.10"
 tar = "0.4"
 toml = "0.5"
 ignore = "0.4.10"

src/bootstrap/builder.rs

@@ -879,7 +879,6 @@ impl<'a> Builder<'a> {
     ) {
         // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
         let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
-        // FIXME: support `do_verify` (only really needed for nightly rustfmt)
         self.download_with_retries(&tempfile, &format!("{}/{}", base, url), help_on_error);
         t!(std::fs::rename(&tempfile, dest_path));
     }
@@ -971,6 +970,28 @@ impl<'a> Builder<'a> {
         t!(fs::remove_dir_all(dst.join(directory_prefix)));
     }
 
+    /// Returns whether the SHA256 checksum of `path` matches `expected`.
+    pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool {
+        use sha2::Digest;
+
+        self.verbose(&format!("verifying {}", path.display()));
+        let mut hasher = sha2::Sha256::new();
+        // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components.
+        // Consider using streaming IO instead?
+        let contents = if self.config.dry_run { vec![] } else { t!(fs::read(path)) };
+        hasher.update(&contents);
+        let found = hex::encode(hasher.finalize().as_slice());
+        let verified = found == expected;
+        if !verified && !self.config.dry_run {
+            println!(
+                "invalid checksum: \n\
+                found:    {found}\n\
+                expected: {expected}",
+            );
+        }
+        return verified;
+    }
+
     /// Obtain a compiler at a given stage and for a given host. Explicitly does
     /// not take `Compiler` since all `Compiler` instances are meant to be
     /// obtained through this function, since it ensures that they are valid

src/bootstrap/config.rs

@@ -1486,6 +1486,7 @@ fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
     #[derive(Deserialize)]
     struct Stage0Metadata {
         dist_server: String,
+        checksums_sha256: HashMap<String, String>,
         rustfmt: Option<RustfmtMetadata>,
     }
     #[derive(Deserialize)]
@@ -1495,10 +1496,11 @@ fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
     }
 
     let stage0_json = builder.read(&builder.src.join("src").join("stage0.json"));
-    let metadata = t!(serde_json::from_str::<Stage0Metadata>(&stage0_json));
-    let RustfmtMetadata { date, version } = metadata.rustfmt?;
+    let Stage0Metadata { dist_server, checksums_sha256, rustfmt } =
+        t!(serde_json::from_str::<Stage0Metadata>(&stage0_json));
+    let RustfmtMetadata { date, version } = rustfmt?;
     let channel = format!("{version}-{date}");
-    let mut dist_server = env::var("RUSTUP_DIST_SERVER").unwrap_or(metadata.dist_server);
+    let mut dist_server = env::var("RUSTUP_DIST_SERVER").unwrap_or(dist_server);
     dist_server.push_str("/dist");
 
     let host = builder.config.build;
@@ -1510,8 +1512,15 @@ fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
     }
 
     let filename = format!("rustfmt-{version}-{build}.tar.xz", build = host.triple);
-    download_component(builder, &dist_server, filename, "rustfmt-preview", &date, "stage0");
-    assert!(rustfmt_path.exists());
+    download_component(
+        builder,
+        &dist_server,
+        filename,
+        "rustfmt-preview",
+        &date,
+        "stage0",
+        Some(checksums_sha256),
+    );
 
     builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt"));
     builder.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt"));
@@ -1564,6 +1573,7 @@ fn download_ci_component(builder: &Builder<'_>, filename: String, prefix: &str,
         prefix,
         commit,
         "ci-rustc",
+        None,
     )
 }
 
@@ -1574,17 +1584,55 @@ fn download_component(
     prefix: &str,
     key: &str,
     destination: &str,
+    checksums: Option<HashMap<String, String>>,
 ) {
     let cache_dst = builder.out.join("cache");
     let cache_dir = cache_dst.join(key);
     if !cache_dir.exists() {
         t!(fs::create_dir_all(&cache_dir));
     }
 
+    let bin_root = builder.out.join(builder.config.build.triple).join(destination);
     let tarball = cache_dir.join(&filename);
-    if !tarball.exists() {
-        builder.download_component(base_url, &format!("{key}/{filename}"), &tarball, "");
+    let url = format!("{key}/{filename}");
+
+    // For the beta compiler, put special effort into ensuring the checksums are valid.
+    // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update
+    // this on each and every nightly ...
+    let checksum = if let Some(checksums) = &checksums {
+        let error = format!(
+            "src/stage0.json doesn't contain a checksum for {url}. \
+            Pre-built artifacts might not be available for this \
+            target at this time, see https://doc.rust-lang.org/nightly\
+            /rustc/platform-support.html for more information."
+        );
+        // TODO: add an enum { Commit, Published } so we don't have to hardcode `dist` in two places
+        let sha256 = checksums.get(&format!("dist/{url}")).expect(&error);
+        if tarball.exists() {
+            if builder.verify(&tarball, sha256) {
+                builder.unpack(&tarball, &bin_root, prefix);
+                return;
+            } else {
+                builder.verbose(&format!(
+                    "ignoring cached file {} due to failed verification",
+                    tarball.display()
+                ));
+                builder.remove(&tarball);
+            }
+        }
+        Some(sha256)
+    } else if tarball.exists() {
+        return;
+    } else {
+        None
+    };
+
+    builder.download_component(base_url, &url, &tarball, "");
+    if let Some(sha256) = checksum {
+        if !builder.verify(&tarball, sha256) {
+            panic!("failed to verify {}", tarball.display());
+        }
     }
-    let bin_root = builder.out.join(builder.config.build.triple).join(destination);
-    builder.unpack(&tarball, &bin_root, prefix)
+
+    builder.unpack(&tarball, &bin_root, prefix);
 }