Add retries to tool installs (#546)

# Summary

We often encounter setup failures in the CI.
This PR adds retries to mitigate these transient installation failures.

## Proof of Work

CI is green, and we see less setup failures

Author: Julien-Ben

.evergreen.yml

@@ -1389,7 +1389,7 @@ buildvariants:
     display_name: e2e_operator_race_ubi_with_telemetry
     tags: [ "pr_patch", "staging", "e2e_test_suite" ]
     run_on:
-      - ubuntu1804-xlarge
+      - ubuntu2404-xlarge
     <<: *base_om7_dependency_with_race
     tasks:
       - name: e2e_operator_race_with_telemetry_task_group
@@ -1666,7 +1666,7 @@ buildvariants:
     tags: [ "pr_patch", "staging", "e2e_mco_test_suite" ]
     <<: *community_dependency
     run_on:
-      - ubuntu2004-large
+      - ubuntu2404-large
 
   ## Manual (patch) E2E tests not run for every PR and commit
 

scripts/evergreen/lint_code.sh

@@ -7,7 +7,7 @@ required_version="v2.0.2"
 # Install or update golangci-lint if not installed or version is incorrect
 if ! [[ -x "$(command -v golangci-lint)" ]]; then
     echo "Installing/updating golangci-lint to version ${required_version}..."
-    curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)"/bin "${required_version}"
+    curl --retry 5 --retry-delay 3 --retry-all-errors --fail --show-error --max-time 180 -sSL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)"/bin "${required_version}"
 else
     echo "golangci-lint is already installed"
 fi

scripts/evergreen/setup_aws.sh

@@ -19,7 +19,7 @@ install_aws_cli_binary() {
     cd "${temp_dir}"
 
     echo "Downloading AWS CLI v2 for ${aws_arch}..."
-    curl -s "https://awscli.amazonaws.com/awscli-exe-linux-${aws_arch}.zip" -o "awscliv2.zip"
+    curl --retry 5 --retry-delay 3 --retry-all-errors --fail --show-error --max-time 180 -s "https://awscli.amazonaws.com/awscli-exe-linux-${aws_arch}.zip" -o "awscliv2.zip"
 
     unzip -q awscliv2.zip
     sudo ./aws/install --update

scripts/evergreen/setup_gcloud_cli.sh

@@ -3,7 +3,7 @@ set -Eeou pipefail
 
 source scripts/dev/set_env_context.sh
 
-curl -s --retry 3 -LO "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-linux-x86_64.tar.gz"
+curl --retry 5 --retry-delay 3 --retry-all-errors --fail --show-error --max-time 180 --silent -LO "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-linux-x86_64.tar.gz"
 tar xf google-cloud-cli-linux-x86_64.tar.gz -C "${workdir}"
 "${workdir}"/google-cloud-sdk/install.sh --quiet
 source "${workdir}/google-cloud-sdk/path.bash.inc"

scripts/evergreen/setup_kubectl.sh

@@ -25,7 +25,7 @@ mv kubectl "${bindir}"
 echo "Downloading helm for ${ARCH}"
 helm_archive="${tmpdir}/helm.tgz"
 helm_version="v3.17.1"
-curl -s https://get.helm.sh/helm-${helm_version}-linux-"${ARCH}".tar.gz --output "${helm_archive}"
+curl --retry 5 --retry-delay 3 --retry-all-errors --fail --show-error --max-time 180 -s https://get.helm.sh/helm-${helm_version}-linux-"${ARCH}".tar.gz --output "${helm_archive}"
 
 tar xfz "${helm_archive}" -C "${tmpdir}" &> /dev/null
 mv "${tmpdir}/linux-${ARCH}/helm" "${bindir}"

scripts/evergreen/setup_mongosh.sh

@@ -4,10 +4,12 @@ set -Eeou pipefail
 source scripts/dev/set_env_context.sh
 
 bindir="${workdir:?}/bin"
-mkdir -p "${bindir}"
+tmpdir="${workdir:?}/tmp"
+mkdir -p "${bindir}" "${tmpdir}"
 
-curl -s --retry 3 -LO "https://downloads.mongodb.com/compass/mongosh-2.3.8-linux-x64.tgz"
-tar -zxvf mongosh-2.3.8-linux-x64.tgz
-cd mongosh-2.3.8-linux-x64/bin
+# Download mongosh archive
+curl --retry 5 --retry-delay 3 --retry-all-errors --fail --show-error --max-time 180 --silent -LO "https://downloads.mongodb.com/compass/mongosh-2.3.8-linux-x64.tgz"
+tar -zxvf mongosh-2.3.8-linux-x64.tgz -C "${tmpdir}"
+cd "${tmpdir}/mongosh-2.3.8-linux-x64/bin"
 ./mongosh --version
 mv mongosh "${bindir}"

scripts/evergreen/setup_prepare_openshift_bundles.sh

@@ -33,7 +33,7 @@ fi
 
 # there is no mac build in for arm64
 opm_arch="amd64"
-curl -L --retry 3 -o opm.tar.gz "https://mirror.openshift.com/pub/openshift-v4/${opm_arch}/clients/ocp/latest-4.12/opm-${opm_os}.tar.gz"
+curl --retry 5 --retry-delay 3 --retry-all-errors --fail --show-error --max-time 180 -L -o opm.tar.gz "https://mirror.openshift.com/pub/openshift-v4/${opm_arch}/clients/ocp/latest-4.12/opm-${opm_os}.tar.gz"
 
 # TODO: Sometimes tar is failing for unknown reasons in EVG. This is left intentionally. Remove if not causing problems anymore.
 ls -al opm.tar.gz

scripts/evergreen/setup_shellcheck.sh

@@ -10,7 +10,7 @@ mkdir -p "${bindir}" "${tmpdir}"
 echo "Downloading shellcheck"
 shellcheck_archive="${tmpdir}/shellcheck.tar.xz"
 shellcheck_version="v0.9.0"
-curl --retry 3 --silent -L "https://github.com/koalaman/shellcheck/releases/download/${shellcheck_version}/shellcheck-${shellcheck_version}.linux.x86_64.tar.xz" -o "${shellcheck_archive}"
+curl --retry 5 --retry-delay 3 --retry-all-errors --fail --show-error --max-time 180 --silent -L "https://github.com/koalaman/shellcheck/releases/download/${shellcheck_version}/shellcheck-${shellcheck_version}.linux.x86_64.tar.xz" -o "${shellcheck_archive}"
 tar -xf "${shellcheck_archive}" -C "${tmpdir}"
 mv "${tmpdir}/shellcheck-${shellcheck_version}/shellcheck" "${bindir}"
 rm "${shellcheck_archive}"

scripts/funcs/install

@@ -46,7 +46,7 @@ download_and_install_binary() {
 
   mkdir -p "${dir}"
   echo "Downloading ${url}"
-  curl --retry 3 --silent -L "${url}" -o "${bin}"
+  curl --retry 5 --retry-delay 3 --retry-all-errors --fail --show-error --max-time 180 --silent -L "${url}" -o "${bin}"
   chmod +x "${bin}"
   mv "${bin}" "${dir}"
   echo "Installed ${bin} to ${dir}"