Rollup merge of #45209 - kennytm:treat-checksum-error-as-download-error, r=Mark-Simulacrum

rustbuild: Make openssl download more reliable.

1. Add `-f` flag to curl, so when the server returns 403 or 500 it will fail immediately.
2. Moved the checksum part into the retry loop, assuming checksum failure is due to broken download that can be fixed by downloading again.

This PR is created responding to two recent spurious failures in #45075 (comment) and #45030 (comment).

r? @Mark-Simulacrum , cc @aidanhs

Author: kennytm

src/bootstrap/native.rs

@@ -352,34 +352,51 @@ impl Step for Openssl {
             // originally from https://www.openssl.org/source/...
             let url = format!("https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/{}",
                               name);
-            let mut ok = false;
+            let mut last_error = None;
             for _ in 0..3 {
                 let status = Command::new("curl")
                                 .arg("-o").arg(&tmp)
+                                .arg("-f")  // make curl fail if the URL does not return HTTP 200
                                 .arg(&url)
                                 .status()
                                 .expect("failed to spawn curl");
-                if status.success() {
-                    ok = true;
-                    break
+
+                // Retry if download failed.
+                if !status.success() {
+                    last_error = Some(status.to_string());
+                    continue;
                 }
+
+                // Ensure the hash is correct.
+                let mut shasum = if target.contains("apple") || build.build.contains("netbsd") {
+                    let mut cmd = Command::new("shasum");
+                    cmd.arg("-a").arg("256");
+                    cmd
+                } else {
+                    Command::new("sha256sum")
+                };
+                let output = output(&mut shasum.arg(&tmp));
+                let found = output.split_whitespace().next().unwrap();
+
+                // If the hash is wrong, probably the download is incomplete or S3 served an error
+                // page. In any case, retry.
+                if found != OPENSSL_SHA256 {
+                    last_error = Some(format!(
+                        "downloaded openssl sha256 different\n\
+                         expected: {}\n\
+                         found:    {}\n",
+                        OPENSSL_SHA256,
+                        found
+                    ));
+                    continue;
+                }
+
+                // Everything is fine, so exit the retry loop.
+                last_error = None;
+                break;
             }
-            if !ok {
-                panic!("failed to download openssl source")
-            }
-            let mut shasum = if target.contains("apple") || build.build.contains("netbsd") {
-                let mut cmd = Command::new("shasum");
-                cmd.arg("-a").arg("256");
-                cmd
-            } else {
-                Command::new("sha256sum")
-            };
-            let output = output(&mut shasum.arg(&tmp));
-            let found = output.split_whitespace().next().unwrap();
-            if found != OPENSSL_SHA256 {
-                panic!("downloaded openssl sha256 different\n\
-                        expected: {}\n\
-                        found:    {}\n", OPENSSL_SHA256, found);
+            if let Some(error) = last_error {
+                panic!("failed to download openssl source: {}", error);
             }
             t!(fs::rename(&tmp, &tarball));
         }