Review fixes

Author: MaciejKaras

docker/mongodb-kubernetes-tests/Dockerfile

@@ -16,7 +16,11 @@ RUN apt-get -qq update \
 
 COPY requirements.txt requirements.txt
 
-RUN python3 -m venv /venv && . /venv/bin/activate && pip install --upgrade pip && GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 pip install -r requirements.txt
+RUN python3 -m venv /venv  \
+    && . /venv/bin/activate  \
+    && pip install --upgrade pip  \
+    && GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 pip install -r requirements.txt \
+    && pip install awscli # install aws, required to run helm registry login while running the tests
 
 FROM scratch AS tools_downloader
 
@@ -44,9 +48,7 @@ RUN apt-get -qq update \
     curl \
     libldap2-dev \
     libsasl2-dev \
-    git \
-    openssl \
-    unzip
+    git
 
 RUN mkdir -p /tmp/mongodb-tools && \
     tar xfz /tmp/mongodb-tools.tgz -C /tmp/mongodb-tools && \
@@ -67,11 +69,6 @@ RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -
     && chmod +x ./kubectl \
     && mv ./kubectl /usr/local/bin/kubectl
 
-# install aws, required to run helm registry login while running the tests
-RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
-    && unzip -q awscliv2.zip \
-    && ./aws/install
-
 COPY --from=builder /venv /venv
 
 ENV PATH="/venv/bin:${PATH}"

docker/mongodb-kubernetes-tests/kubetester/helm.py

@@ -149,7 +149,7 @@ def process_run_and_check(args, **kwargs):
         raise
 
 
-def oci_helm_registry_login(helm_registry: str, region: str):
+def helm_registry_login_to_ecr(helm_registry: str, region: str):
     logger.info(f"Attempting to log into ECR registry: {helm_registry}, using helm registry login.")
 
     aws_command = ["aws", "ecr", "get-login-password", "--region", region]
@@ -303,17 +303,17 @@ def _helm_chart_dir(default: Optional[str] = "helm_chart") -> str:
 
 
 # helm_chart_path_and_version returns the chart path and version that we would like to install to run the E2E tests.
-# for local tests it returns early with local helm chart dir and for other scenarios it figure out the chart and version
+# for local tests it returns early with local helm chart dir and for other scenarios it figures out the chart and version
 # based on the caller. In most of the cases we will install chart from OCI registry but for the tests where we would like
-# to install MEKO's specific version or MCK's specific version, we would expec `helm_chart_path` to set already.
+# to install MEKO's specific version or MCK's specific version, we would expect `helm_chart_path` to set already.
 def helm_chart_path_and_version(helm_chart_path: str, operator_version: str) -> tuple[str, str]:
     # these are imported here to resolve import cycle issue
     from tests.conftest import LOCAL_HELM_CHART_DIR, local_operator
 
     if local_operator():
         return LOCAL_HELM_CHART_DIR, ""
 
-    # if operator_version is not specified and we are not installing the MCK or MEKO chart
+    # if operator_version is not specified, and we are not installing the MCK or MEKO chart
     # it would mean we want to install OCI published helm chart. Figure out respective version,
     # it is set in env var `OPERATOR_VERSION` based on build_scenario.
     if not operator_version and helm_chart_path not in (
@@ -328,12 +328,13 @@ def helm_chart_path_and_version(helm_chart_path: str, operator_version: str) ->
     # helm_chart_path not being passed would mean we are on evg env and would like to
     # install helm chart from OCI registry.
     if not helm_chart_path:
-        # login to the OCI container registry
         registry, repository, region = oci_chart_info()
-        try:
-            oci_helm_registry_login(registry, region)
-        except Exception as e:
-            raise e
+        # If ECR we need to login first to the OCI container registry
+        if registry == "268558157000.dkr.ecr.us-east-1.amazonaws.com":
+            try:
+                helm_registry_login_to_ecr(registry, region)
+            except Exception as e:
+                raise Exception(f"Failed to login to ECR helm registry {registry}. Error: {e}")
 
         # figure out the registry URI, based on dev/staging scenario
         chart_uri = f"oci://{registry}/{repository}"

docker/mongodb-kubernetes-tests/kubetester/operator.py

@@ -1,7 +1,5 @@
 from __future__ import annotations
 
-import logging
-import os
 import time
 from typing import Dict, List, Optional
 
@@ -10,7 +8,6 @@
 from kubernetes.client import V1CustomResourceDefinition, V1Deployment, V1Pod
 from kubernetes.client.rest import ApiException
 from kubetester import wait_for_webhook
-from kubetester.consts import *
 from kubetester.create_or_replace_from_yaml import create_or_replace_from_yaml
 from kubetester.helm import (
     helm_chart_path_and_version,
@@ -19,8 +16,6 @@
     helm_template,
     helm_uninstall,
     helm_upgrade,
-    oci_chart_info,
-    oci_helm_registry_login,
 )
 from tests import test_logger
 

docker/mongodb-kubernetes-tests/tests/conftest.py

@@ -22,11 +22,8 @@
 from kubetester.awss3client import AwsS3Client
 from kubetester.consts import *
 from kubetester.helm import (
-    helm_chart_path_and_version,
     helm_install_from_chart,
     helm_repo_add,
-    oci_chart_info,
-    oci_helm_registry_login,
 )
 from kubetester.kubetester import KubernetesTester
 from kubetester.kubetester import fixture as _fixture
@@ -840,8 +837,6 @@ def _install_multi_cluster_operator(
     # The Operator will be installed from the following repo, so adding it first
     helm_repo_add("mongodb", "https://mongodb.github.io/helm-charts")
 
-    helm_chart_path, custom_operator_version = helm_chart_path_and_version(helm_chart_path, custom_operator_version)
-
     prepare_multi_cluster_namespaces(
         namespace,
         multi_cluster_operator_installation_config,
@@ -857,6 +852,7 @@ def _install_multi_cluster_operator(
         helm_args=multi_cluster_operator_installation_config,
         api_client=central_cluster_client,
         helm_chart_path=helm_chart_path,
+        operator_version=custom_operator_version,
     ).upgrade(multi_cluster=True, custom_operator_version=custom_operator_version, apply_crds_first=apply_crds_first)
 
     # If we're running locally, then immediately after installing the deployment, we scale it to zero.

scripts/evergreen/e2e/single_e2e.sh

@@ -48,14 +48,15 @@ deploy_test_app() {
     BUILD_ID="${BUILD_ID:-default_build_id}"
     BUILD_VARIANT="${BUILD_VARIANT:-default_build_variant}"
 
-    chart_info=$(scripts/dev/run_python.sh scripts/release/oci_chart_info.py)
-    helm_oci_regisry=$(echo "${chart_info}" | jq -r '.registry' )
+    chart_info=$(scripts/dev/run_python.sh scripts/release/oci_chart_info.py --build_scenario "${BUILD_SCENARIO}")
+    helm_oci_registry=$(echo "${chart_info}" | jq -r '.registry' )
     helm_oci_repository=$(echo "${chart_info}" | jq -r '.repository' )
     helm_oci_registry_region=$(echo "${chart_info}" | jq -r '.region' )
 
     # note, that the 4 last parameters are used only for Mongodb resource testing - not for Ops Manager
     helm_params=(
         "--set" "taskId=${task_id:-'not-specified'}"
+        "--set" "operator.version=${OPERATOR_VERSION}"
         "--set" "namespace=${NAMESPACE}"
         "--set" "taskName=${task_name}"
         "--set" "mekoTestsRegistry=${MEKO_TESTS_REGISTRY}"
@@ -81,6 +82,9 @@ deploy_test_app() {
         "--set" "cognito_user_password=${cognito_user_password}"
         "--set" "cognito_workload_url=${cognito_workload_url}"
         "--set" "cognito_workload_user_id=${cognito_workload_user_id}"
+        "--set" "helm.oci.registry=${helm_oci_registry}"
+        "--set" "helm.oci.repository=${helm_oci_repository}"
+        "--set" "helm.oci.region=${helm_oci_registry_region}"
     )
 
     # shellcheck disable=SC2154
@@ -147,22 +151,6 @@ deploy_test_app() {
         helm_params+=("--set" "omDebugHttp=true")
     fi
 
-    if [[ -n "${helm_oci_regisry:-}" ]]; then
-        helm_params+=("--set" "helm.oci.registry=${helm_oci_regisry}")
-    fi
-
-    if [[ -n "${helm_oci_repository:-}" ]]; then
-        helm_params+=("--set" "helm.oci.repository=${helm_oci_repository}")
-    fi
-
-    if [[ -n "${helm_oci_registry_region:-}" ]]; then
-        helm_params+=("--set" "helm.oci.region=${helm_oci_registry_region}")
-    fi
-
-    if [[ -n "${OPERATOR_VERSION:-}" ]]; then
-        helm_params+=("--set" "operator.version=${OPERATOR_VERSION}")
-    fi
-
     helm_params+=("--set" "opsManagerVersion=${ops_manager_version}")
 
     helm template "scripts/evergreen/deployments/test-app" "${helm_params[@]}" > "${helm_template_file}" || exit 1

scripts/release/oci_chart_info.py

@@ -1,14 +1,33 @@
+import argparse
 import json
-import os
-import sys
 from dataclasses import asdict
 
-from lib.base_logger import logger
+from release.build.build_scenario import BuildScenario
+
 from scripts.release.build.build_info import load_build_info
 
 
 def main():
-    build_scenario = os.environ.get("BUILD_SCENARIO")
+    supported_scenarios = list(BuildScenario)
+
+    parser = argparse.ArgumentParser(
+        description="""Dump the Helm chart information for the 'mongodb-kubernetes' chart as JSON.""",
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    parser.add_argument(
+        "-b",
+        "--build-scenario",
+        metavar="",
+        action="store",
+        required=True,
+        type=str,
+        choices=supported_scenarios,
+        help=f"""Build scenario when reading configuration from 'build_info.json'.
+    Options: {", ".join(supported_scenarios)}. For '{BuildScenario.DEVELOPMENT}' the '{BuildScenario.PATCH}' scenario is used to read values from 'build_info.json'""",
+    )
+    args = parser.parse_args()
+
+    build_scenario = BuildScenario(args.build_scenario)
     build_info = load_build_info(build_scenario)
     chart_info = build_info.helm_charts["mongodb-kubernetes"]
 
@@ -20,5 +39,4 @@ def main():
     try:
         main()
     except Exception as e:
-        logger.error(f"Failed while dumping the chart_info as json. Error: {e}")
-        sys.exit(1)
+        raise Exception(f"Failed while dumping the chart_info as json. Error: {e}")