CLOUDP-339241 - Improve e2e on arm (#515)

# Summary

This PR adds a new task running on arm64 to build the arm64 test image.
Previously it was built using emulation on an amd64 resulting in a
runtime of ~40 minutes.

## Proof of Work

CI is green
[Smoke test on
ARM](https://spruce.mongodb.com/version/68ee6adf9eafcd00078b8b26/tasks?sorts=STATUS%3AASC%3BBASE_STATUS%3ADESC)
also green

## Checklist

- [x] Have you linked a jira ticket and/or is the ticket in the title?
- [x] Have you checked whether your jira ticket required DOCSP changes?
- [x] Have you added changelog file?
    - use `skip-changelog` label if not needed
- refer to [Changelog files and Release
Notes](https://github.com/mongodb/mongodb-kubernetes/blob/master/CONTRIBUTING.md#changelog-files-and-release-notes)
section in CONTRIBUTING.md for more details

Author: lucian-tosa

.evergreen-functions.yml

@@ -527,6 +527,7 @@ functions:
         binary: scripts/release/pipeline.sh
         env:
           IMAGE_NAME: ${IMAGE_NAME}
+          FLAGS: ${FLAGS}
 
   pipeline_agent:
     - command: subprocess.exec

.evergreen.yml

@@ -351,6 +351,16 @@ tasks:
         vars:
           IMAGE_NAME: meko-tests
 
+  - name: build_test_image_arm
+    commands:
+      - func: clone
+      - func: setup_building_host
+      - func: build_multi_cluster_binary
+      - func: pipeline
+        vars:
+          IMAGE_NAME: meko-tests-arm64
+
+
   - name: build_test_image_ibm
     commands:
       - func: clone
@@ -1386,10 +1396,6 @@ buildvariants:
         variant: init_test_run
       - name: build_database_image_ubi
         variant: init_test_run
-      - name: build_init_appdb_images_ubi
-        variant: init_test_run
-      - name: build_init_om_images_ubi
-        variant: init_test_run
       - name: build_test_image_ibm
         variant: init_test_run_ibm_power
     tasks:
@@ -1412,10 +1418,6 @@ buildvariants:
         variant: init_test_run
       - name: build_database_image_ubi
         variant: init_test_run
-      - name: build_init_appdb_images_ubi
-        variant: init_test_run
-      - name: build_init_om_images_ubi
-        variant: init_test_run
       - name: build_test_image_ibm
         variant: init_test_run_ibm_power
     tasks:
@@ -1438,10 +1440,6 @@ buildvariants:
         variant: init_test_run
       - name: build_database_image_ubi
         variant: init_test_run
-      - name: build_init_appdb_images_ubi
-        variant: init_test_run
-      - name: build_init_om_images_ubi
-        variant: init_test_run
       - name: build_test_image_ibm
         variant: init_test_run_ibm_z
     tasks:
@@ -1462,12 +1460,6 @@ buildvariants:
         variant: init_test_run
       - name: build_init_database_image_ubi
         variant: init_test_run
-      - name: build_database_image_ubi
-        variant: init_test_run
-      - name: build_init_appdb_images_ubi
-        variant: init_test_run
-      - name: build_init_om_images_ubi
-        variant: init_test_run
       - name: build_test_image_ibm
         variant: init_test_run_ibm_z
     tasks:
@@ -1479,10 +1471,18 @@ buildvariants:
     run_on:
       - ubuntu2404-arm64-large
     allowed_requesters: [ "patch", "commit" ]
+    depends_on:
+      - name: build_operator_ubi
+        variant: init_test_run
+      - name: build_init_database_image_ubi
+        variant: init_test_run
+      - name: build_database_image_ubi
+        variant: init_test_run
+      - name: build_test_image_arm
+        variant: init_test_run_arm
     # TODO: Re-enable when staging is added to pipeline
     # https://jira.mongodb.org/browse/CLOUDP-349096
     disable: true
-    <<: *base_no_om_image_dependency
     tasks:
       - name: e2e_smoke_arm_task_group
 
@@ -1492,10 +1492,16 @@ buildvariants:
     run_on:
       - ubuntu2404-arm64-large
     allowed_requesters: [ "patch", "commit" ]
+    depends_on:
+      - name: build_operator_ubi
+        variant: init_test_run
+      - name: build_init_database_image_ubi
+        variant: init_test_run
+      - name: build_test_image_arm
+        variant: init_test_run_arm
     # TODO: Re-enable when staging is added to pipeline
     # https://jira.mongodb.org/browse/CLOUDP-349096
     disable: true
-    <<: *base_no_om_image_dependency
     tasks:
       - name: e2e_smoke_arm_task_group
 
@@ -1727,6 +1733,15 @@ buildvariants:
     tasks:
       - name: build_test_image_ibm
 
+  - name: init_test_run_arm
+    display_name: init_test_run_arm
+    max_hosts: -1
+    tags: [ "staging" ]
+    run_on:
+      - ubuntu2204-arm64-small
+    tasks:
+      - name: build_test_image_arm
+
   - name: run_pre_commit
     priority: 70
     display_name: run_pre_commit

build_info.json

@@ -180,7 +180,6 @@
       "staging": {
         "repositories": ["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests"],
         "platforms": [
-          "linux/arm64",
           "linux/amd64"
         ]
       },
@@ -192,6 +191,27 @@
         ]
       }
     },
+    "meko-tests-arm64": {
+      "dockerfile-path": "docker/mongodb-kubernetes-tests/Dockerfile",
+      "patch": {
+        "repositories": [
+          "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-tests"
+        ],
+        "platforms": [
+          "linux/arm64"
+        ],
+        "architecture_suffix": true
+      },
+      "staging": {
+        "repositories": [
+          "268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests"
+        ],
+        "platforms": [
+          "linux/arm64"
+        ],
+        "architecture_suffix": true
+      }
+    },
     "mco-tests": {
       "dockerfile-path": "docker/mongodb-community-tests/Dockerfile",
       "patch": {

scripts/dev/setup_evg_host.sh

@@ -47,6 +47,7 @@ download_kind() {
 download_kubectl() {
   kubectl_version=$(curl --retry 5 -Ls https://dl.k8s.io/release/stable.txt)
   echo "Downloading kubectl ${kubectl_version}..."
+  kubectl_version=$(echo "${kubectl_version}" | tail -n1 | tr -d '\n')
 
   curl --retry 5 -LOs "https://dl.k8s.io/release/${kubectl_version}/bin/linux/${ARCH}/kubectl"
   chmod +x kubectl

scripts/evergreen/e2e/single_e2e.sh

@@ -28,14 +28,17 @@ deploy_test_app() {
     arch=$(uname -m)
 
     case "${arch}" in
+        aarch64|arm64)
+            meko_tests_version="${meko_tests_version}-arm64"
+            ;;
         ppc64le)
             meko_tests_version="${meko_tests_version}-ppc64le"
             ;;
         s390x)
             meko_tests_version="${meko_tests_version}-s390x"
             ;;
         *)
-            echo "Not IBM host, using default meko_tests_version"
+            echo "amd64 host, using default meko_tests_version"
             ;;
     esac
 

scripts/evergreen/setup_kubectl.sh

@@ -14,6 +14,7 @@ mkdir -p "${bindir}" "${tmpdir}"
 
 kubectl_version=$(curl --retry 5 -Ls https://dl.k8s.io/release/stable.txt)
 echo "Downloading kubectl ${kubectl_version} for ${ARCH}"
+kubectl_version=$(echo "${kubectl_version}" | tail -n1 | tr -d '\n')
 
 curl --retry 5 -LOs "https://dl.k8s.io/release/${kubectl_version}/bin/linux/${ARCH}/kubectl"
 chmod +x kubectl

scripts/release/atomic_pipeline.py

@@ -87,6 +87,9 @@ def build_image(
             if build_configuration.olm_tag:
                 olm_tag = create_olm_version_tag(build_configuration.version)
                 tags.append(f"{registry}:{olm_tag}")
+            if build_configuration.architecture_suffix and len(build_configuration.platforms) == 1:
+                arch = build_configuration.platforms[0].split("/")[1]
+                tags.append(f"{tag}-{arch}")
 
     if not tags:
         logger.info("All specified image tags already exist. Skipping build.")

scripts/release/build/build_info.py

@@ -5,6 +5,7 @@
 from scripts.release.build.build_scenario import BuildScenario
 
 MEKO_TESTS_IMAGE = "meko-tests"
+MEKO_TESTS_ARM64_IMAGE = "meko-tests-arm64"
 OPERATOR_IMAGE = "operator"
 OPERATOR_RACE_IMAGE = "operator-race"
 MCO_TESTS_IMAGE = "mco-tests"
@@ -27,6 +28,7 @@ class ImageInfo:
     latest_tag: bool = False
     olm_tag: bool = False
     skip_if_exists: bool = False
+    architecture_suffix: bool = False
 
 
 @dataclass
@@ -80,6 +82,7 @@ def load_build_info(scenario: BuildScenario) -> BuildInfo:
             latest_tag=scenario_data.get("latest-tag", False),
             olm_tag=scenario_data.get("olm-tag", False),
             skip_if_exists=scenario_data.get("skip-if-exists", False),
+            architecture_suffix=scenario_data.get("architecture_suffix", False),
         )
 
     binaries = {}

scripts/release/build/image_build_configuration.py

@@ -23,6 +23,7 @@ class ImageBuildConfiguration:
     parallel_factor: int = 0
     all_agents: bool = False
     currently_used_agents: bool = False
+    architecture_suffix: bool = False
 
     def is_release_scenario(self) -> bool:
         return self.scenario == BuildScenario.RELEASE

scripts/release/pipeline.py

@@ -37,6 +37,7 @@
     INIT_DATABASE_IMAGE,
     INIT_OPS_MANAGER_IMAGE,
     MCO_TESTS_IMAGE,
+    MEKO_TESTS_ARM64_IMAGE,
     MEKO_TESTS_IMAGE,
     OPERATOR_IMAGE,
     OPERATOR_RACE_IMAGE,
@@ -69,6 +70,7 @@ def get_builder_function_for_image_name() -> Dict[str, Callable]:
 
     image_builders = {
         MEKO_TESTS_IMAGE: build_meko_tests_image,
+        MEKO_TESTS_ARM64_IMAGE: build_meko_tests_image,
         OPERATOR_IMAGE: build_operator_image,
         OPERATOR_RACE_IMAGE: partial(build_operator_image, with_race_detection=True),
         MCO_TESTS_IMAGE: build_mco_tests_image,
@@ -120,6 +122,12 @@ def image_build_config_from_args(args) -> ImageBuildConfiguration:
     platforms = get_platforms_from_arg(args.platform) or image_build_info.platforms
     sign = args.sign if args.sign is not None else image_build_info.sign
     skip_if_exists = args.skip_if_exists if args.skip_if_exists is not None else image_build_info.skip_if_exists
+    architecture_suffix = (
+        args.architecture_suffix if args.architecture_suffix is not None else image_build_info.architecture_suffix
+    )
+
+    if architecture_suffix and len(platforms) > 1:
+        raise ValueError("Cannot use architecture suffix with multi-platform builds")
 
     # Validate version - only agent can have None version as the versions are managed by the agent
     # which are externally retrieved from release.json
@@ -140,6 +148,7 @@ def image_build_config_from_args(args) -> ImageBuildConfiguration:
         parallel_factor=args.parallel_factor,
         all_agents=args.all_agents,
         currently_used_agents=args.current_agents,
+        architecture_suffix=architecture_suffix,
     )
 
 
@@ -283,6 +292,11 @@ def main():
         action="store_true",
         help="Build all currently used agent images.",
     )
+    parser.add_argument(
+        "--architecture-suffix",
+        action=argparse.BooleanOptionalAction,
+        help="Append architecture suffix to image tags for single platform builds. Can be true or false. This will override the value from build_info.json",
+    )
 
     args = parser.parse_args()