Update MEKO to MCK OLM tests (#97)

Instead of relying on MEKO -> MCK upgrade path in a catalog metadata, we
now use uninstall-then-install migration approach.

While MEKO -> MCK upgrade path is technically possible, it is not
allowed in operatorhub.io and Red Hat certified operators catalogs as it
is prevented by validation.

Author: m1kola

config/manifests/bases/mongodb-kubernetes.clusterserviceversion.yaml

@@ -443,5 +443,4 @@ spec:
   maturity: stable
   provider:
     name: MongoDB, Inc
-  replaces: mongodb-enterprise.v1.33.0
   version: 0.0.0

docker/mongodb-kubernetes-tests/tests/olm/olm_meko_operator_upgrade_with_resources.py

@@ -16,6 +16,7 @@
 from kubetester.mongodb_user import MongoDBUser
 from kubetester.opsmanager import MongoDBOpsManager
 from pytest import fixture
+from tests import test_logger
 from tests.conftest import LEGACY_OPERATOR_NAME, OPERATOR_NAME
 from tests.olm.olm_test_commons import (
     get_catalog_image,
@@ -28,18 +29,22 @@
     wait_for_operator_ready,
 )
 from tests.opsmanager.om_ops_manager_backup import create_aws_secret, create_s3_bucket
-from tests.upgrades import downscale_operator_deployment
+
+logger = test_logger.get_test_logger(__name__)
 
 # See docs how to run this locally: https://wiki.corp.mongodb.com/display/MMS/E2E+Tests+Notes#E2ETestsNotes-OLMtests
 
 # This test performs operator migration from the latest MEKO to MCK while having OM and MongoDB resources deployed.
-# It performs the following actions:
+# It uses the uninstall-then-install approach since operatorhub.io and other catalogs
+# do not allow cross-package upgrades with the "replaces" directive.
+# The test performs the following actions:
 #  - deploy latest released MEKO operator using OLM
 #  - deploy OM
 #  - deploy backup-required MongoDB: oplog, s3, blockstore
 #  - deploy TLS-enabled sharded MongoDB
 #  - check everything is running
-#  - upgrade the operator to the MCK version built from the current branch
+#  - uninstall MEKO operator by deleting Subscription and ClusterServiceVersion resources
+#  - install MCK operator using OLM with a fresh subscription
 #  - wait for resources to be rolling-updated due to updated stateful sets by the new operator
 #  - check everything is running and connectable
 
@@ -59,15 +64,13 @@ def catalog_source(namespace: str, version_id: str):
 
 
 @fixture
-def subscription(namespace: str, catalog_source: CustomObject):
+def meko_subscription(namespace: str, catalog_source: CustomObject):
     """
-    Create subscription for the operator. The subscription is first created
-    with the latest released version of MEKO operator.
-    Later in the test, it will be updated to MCK.
+    Create subscription for the MEKO operator.
     """
     static_value = get_default_architecture()
     return get_subscription_custom_object(
-        "mongodb-enterprise-operator",
+        LEGACY_OPERATOR_NAME,
         namespace,
         {
             "channel": "stable",  # stable channel contains latest released operator in RedHat's certified repository
@@ -89,6 +92,33 @@ def subscription(namespace: str, catalog_source: CustomObject):
     )
 
 
+def get_mck_subscription_object(namespace: str, catalog_source: CustomObject):
+    """
+    Create a subscription object for the MCK operator.
+    This is a separate function (not a fixture) so it can be called after uninstalling MEKO.
+    """
+    static_value = get_default_architecture()
+    return get_subscription_custom_object(
+        OPERATOR_NAME,
+        namespace,
+        {
+            "channel": "migration",
+            "name": "mongodb-kubernetes",
+            "source": catalog_source.name,
+            "sourceNamespace": namespace,
+            "installPlanApproval": "Automatic",
+            "config": {
+                "env": [
+                    {"name": "MANAGED_SECURITY_CONTEXT", "value": "false"},
+                    {"name": "OPERATOR_ENV", "value": "dev"},
+                    {"name": "MDB_DEFAULT_ARCHITECTURE", "value": static_value},
+                    {"name": "MDB_OPERATOR_TELEMETRY_SEND_ENABLED", "value": "false"},
+                ]
+            },
+        },
+    )
+
+
 @fixture
 def latest_released_meko_version():
     return get_latest_released_operator_version("mongodb-enterprise")
@@ -100,12 +130,10 @@ def test_meko_install_stable_operator_version(
     version_id: str,
     latest_released_meko_version: str,
     catalog_source: CustomObject,
-    subscription: CustomObject,
+    meko_subscription: CustomObject,
 ):
-    subscription.update()
-    wait_for_operator_ready(
-        namespace, "mongodb-enterprise-operator", f"mongodb-enterprise.v{latest_released_meko_version}"
-    )
+    meko_subscription.update()
+    wait_for_operator_ready(namespace, LEGACY_OPERATOR_NAME, f"mongodb-enterprise.v{latest_released_meko_version}")
 
 
 # install resources on the latest released version of the operator
@@ -344,59 +372,124 @@ def test_resources_in_running_state_before_upgrade(
     mdb_sharded.assert_reaches_phase(Phase.Running)
 
 
-# upgrade the operator
+# uninstall MEKO and install MCK operator instead
+
+
+def uninstall_meko_operator(namespace: str, meko_subscription: CustomObject):
+    """Uninstall the MEKO operator by deleting Subscription and ClusterServiceVersion"""
+
+    # Load the subscription from API server
+    # so we can get CSV name from status
+    meko_subscription.load()
+    csv_name = meko_subscription["status"]["installedCSV"]
+
+    # Delete the subscription
+    meko_subscription.delete()
+
+    # Delete ClusterServiceVersion
+    api_instance = kubernetes.client.CustomObjectsApi()
+    api_instance.delete_namespaced_custom_object(
+        group="operators.coreos.com",
+        version="v1alpha1",
+        namespace=namespace,
+        plural="clusterserviceversions",
+        name=csv_name,
+    )
 
 
 @pytest.mark.e2e_olm_meko_operator_upgrade_with_resources
-def test_downscale_meko(namespace: str):
-    # Scale down the existing operator deployment to 0. This is needed as long as the
-    # initial OLM deployment installs the MEKO operator.
-    downscale_operator_deployment(deployment_name=LEGACY_OPERATOR_NAME, namespace=namespace)
+def test_uninstall_meko_operator(
+    namespace: str,
+    meko_subscription: CustomObject,
+):
+    # Uninstall the MEKO operator
+    uninstall_meko_operator(namespace, meko_subscription)
+
+    # Get a list of all statefulsets
+    api_instance = kubernetes.client.AppsV1Api()
+    statefulsets = api_instance.list_namespaced_stateful_set(namespace)
+
+    # Kill one pod from each statefulset to simulate reschedule
+    for sts in statefulsets.items:
+        sts_name = sts.metadata.name
+        logger.info(f"Processing StatefulSet {sts_name}")
+
+        # Get pods for this statefulset
+        if sts.spec.selector and sts.spec.selector.match_labels:
+            # Build label selector string from match_labels dictionary
+            selector_parts = []
+            for key, value in sts.spec.selector.match_labels.items():
+                selector_parts.append(f"{key}={value}")
+            label_selector = ",".join(selector_parts)
+
+            pods = kubernetes.client.CoreV1Api().list_namespaced_pod(namespace, label_selector=label_selector)
+
+            if pods.items:
+                # Delete the first pod
+                pod_name = pods.items[0].metadata.name
+                logger.info(f"Deleting pod {pod_name} from StatefulSet {sts_name}")
+                kubernetes.client.CoreV1Api().delete_namespaced_pod(
+                    name=pod_name, namespace=namespace, body=kubernetes.client.V1DeleteOptions()
+                )
+
+    # Wait for all statefulsets to be ready again
+    def all_statefulsets_ready():
+        for sts in api_instance.list_namespaced_stateful_set(namespace).items:
+            if sts.status.ready_replicas != sts.status.replicas:
+                return (
+                    False,
+                    f"StatefulSet {sts.metadata.name} has {sts.status.ready_replicas}/{sts.status.replicas} ready replicas",
+                )
+        return True, "All StatefulSets are ready"
+
+    run_periodically(
+        all_statefulsets_ready, timeout=600, msg=f"Waiting for all StatefulSets to be ready after pod deletion"
+    )
+
+
+@pytest.mark.e2e_olm_meko_operator_upgrade_with_resources
+def test_connectivity_after_meko_uninstall(
+    ca_path: str,
+    ops_manager: MongoDBOpsManager,
+    oplog_replica_set: MongoDB,
+    blockstore_replica_set: MongoDB,
+    s3_replica_set: MongoDB,
+    mdb_sharded: MongoDB,
+):
+    """Verify resources are still connectable after MEKO operator uninstall but before MCK operator install"""
+    wait_for_om_healthy_response(ops_manager)
+
+    oplog_replica_set.assert_connectivity()
+    blockstore_replica_set.assert_connectivity()
+    s3_replica_set.assert_connectivity()
+    mdb_sharded.assert_connectivity(ca_path=ca_path)
 
 
 @pytest.mark.e2e_olm_meko_operator_upgrade_with_resources
-def test_meko_operator_upgrade_to_mck(
+def test_install_mck_operator(
     namespace: str,
     version_id: str,
     catalog_source: CustomObject,
-    subscription: CustomObject,
 ):
     current_operator_version = get_current_operator_version()
     incremented_operator_version = increment_patch_version(current_operator_version)
 
-    # It is very likely that OLM will be doing a series of status updates during this time.
-    # It's better to employ a retry mechanism and spin here for a while before failing.
-    def update_subscription() -> bool:
-        try:
-            subscription.load()
-            # Update MEKO subscription to MCK
-            subscription["spec"]["name"] = "mongodb-kubernetes"
-            # Migration channel contains operator build from the current branch,
-            # with an upgrade path from the latest MEKO release.
-            subscription["spec"]["channel"] = "migration"
-            subscription.update()
-            return True
-        except kubernetes.client.ApiException as e:
-            if e.status == 409:
-                return False
-        else:
-            raise e
-
-    run_periodically(update_subscription, timeout=100, msg="Subscription to be updated")
+    # Create MCK subscription
+    mck_subscription = get_mck_subscription_object(namespace, catalog_source)
+    mck_subscription.update()
 
     wait_for_operator_ready(namespace, OPERATOR_NAME, f"mongodb-kubernetes.v{incremented_operator_version}")
 
 
 @pytest.mark.e2e_olm_meko_operator_upgrade_with_resources
-def test_one_resources_not_in_running_state(ops_manager: MongoDBOpsManager, mdb_sharded: MongoDB):
-    # Wait for the first resource to become reconciling after operator upgrade.
-    # Only then wait for all to not get a false positive when all resources are ready,
-    # because the upgraded operator haven't started reconciling
+def test_one_resource_not_in_running_state(ops_manager: MongoDBOpsManager):
+    # Wait for the first resource to become reconciling after operator replacement.
+    # This confirms the MCK operator has started reconciling the resources
     ops_manager.om_status().assert_reaches_phase(Phase.Pending, timeout=600)
 
 
 @pytest.mark.e2e_olm_meko_operator_upgrade_with_resources
-def test_resources_in_running_state_after_upgrade(
+def test_resources_in_running_state_after_migration(
     ops_manager: MongoDBOpsManager,
     oplog_replica_set: MongoDB,
     blockstore_replica_set: MongoDB,
@@ -414,7 +507,7 @@ def test_resources_in_running_state_after_upgrade(
 
 
 @pytest.mark.e2e_olm_meko_operator_upgrade_with_resources
-def test_resources_connectivity_after_upgrade(
+def test_resources_connectivity_after_migration(
     ca_path: str,
     ops_manager: MongoDBOpsManager,
     oplog_replica_set: MongoDB,

scripts/evergreen/operator-sdk/prepare-openshift-bundles-for-e2e.sh

@@ -4,17 +4,20 @@ set -Eeou pipefail
 
 # This script prepares and publishes OpenShift bundles and catalog source for operator upgrade tests using OLM.
 # It enables testing two upgrade scenarios:
-#   1. Upgrade from the latest release of MEKO (MongoDB Enterprise Kubernetes Operator)
-#      to the current version of MCK (MongoDB Controllers for Kubernetes).
+#   1. Migration from the latest release of MEKO (MongoDB Enterprise Kubernetes Operator)
+#      to the current version of MCK (MongoDB Controllers for Kubernetes)
+#      by uninstalling MEKO and installing MCK.
 #   2. Upgrade from the latest release of MCK to the current version of MCK.
 #
 # The script builds a test catalog with:
 #   - MEKO package (mongodb-enterprise) with one channel:
 #       - stable channel: latest release of MEKO
-#   - MCK package (mongodb-kubernetes) with two channels:
+#   - MCK package (mongodb-kubernetes) with three channels:
 #       - stable channel: latest release of MCK
 #       - fast channel: current version of MCK with an upgrade path from stable
-#       - migration channel: current version of MCK with an upgrade path from stable of MEKO
+#       - migration channel: current version of MCK. This will be used to test
+#         migration from MEKO to MCK by uninstalling MEKO and installing MCK
+#         so this channel has an entry without the replaces field.
 #
 # Required env vars:
 #   - BASE_REPO_URL (or REGISTRY for EVG run)
@@ -103,7 +106,6 @@ function generate_mck_catalog_metadata() {
   current_bundle_version=$5
   current_bundle_image=$6
   meko_package_name=$7
-  meko_latest_bundle_version=$8
 
   catalog_yaml="${catalog_dir}/${mck_package_name}.yaml"
 
@@ -154,8 +156,7 @@ schema: olm.channel
 package: ${mck_package_name}
 name: migration
 entries:
-  - name: ${mck_package_name}.v${current_bundle_version}
-    replaces: ${meko_package_name}.v${meko_latest_bundle_version}" >>"${catalog_yaml}"
+  - name: ${mck_package_name}.v${current_bundle_version}" >>"${catalog_yaml}"
 }
 
 function generate_meko_catalog_metadata() {
@@ -271,7 +272,7 @@ scripts/evergreen/operator-sdk/prepare-openshift-bundles.sh
 # publish two-channel catalog source to be used in e2e test.
 header "Building and pushing the catalog:"
 catalog_dir="$(init_test_catalog "${certified_repo_cloned}")"
-generate_mck_catalog_metadata "${catalog_dir}" "${mck_package_name}" "${mck_latest_released_operator_version}" "${mck_latest_certified_bundle_image}" "${current_incremented_operator_version_from_release_json}" "${current_bundle_image}" "${meko_package_name}" "${meko_latest_released_operator_version}"
+generate_mck_catalog_metadata "${catalog_dir}" "${mck_package_name}" "${mck_latest_released_operator_version}" "${mck_latest_certified_bundle_image}" "${current_incremented_operator_version_from_release_json}" "${current_bundle_image}" "${meko_package_name}"
 generate_meko_catalog_metadata "${catalog_dir}" "${meko_package_name}" "${meko_latest_released_operator_version}" "${meko_latest_certified_bundle_image}"
 build_and_publish_test_catalog "${catalog_dir}" "${test_catalog_image}"
 

scripts/evergreen/release/update_helm_values_files.py

@@ -94,23 +94,10 @@ def update_cluster_service_version(operator_version):
     image_repo = ":".join(image_parts[:-1])
 
     if old_operator_version != operator_version:
-        olm_package_name = "mongodb-kubernetes"
-        # TODO: CLOUDP-310820 - After 1.0.0 release we need to clean this up: remove this condition
-        if Version(operator_version) <= Version("1.0.0"):
-            # MCK version 1.0.0 is a special case, where we need to
-            # set the olm_package_name to "mongodb-enterprise" because
-            # this is the version which provides a migration path
-            # from the old mongodb-enterprise operator (MEKO)
-            # to the new mongodb-kubernetes (MCK).
-            olm_package_name = "mongodb-enterprise"
-            # This is the latest MEKO version we are going to release.
-            # We hardcode it for now. Later this whole condition will be removed.
-            old_operator_version = "1.33.0"
-
         set_value_in_yaml_file(
             "config/manifests/bases/mongodb-kubernetes.clusterserviceversion.yaml",
             "spec.replaces",
-            f"{olm_package_name}.v{old_operator_version}",
+            f"mongodb-kubernetes.v{old_operator_version}",
             preserve_quotes=True,
         )