CLOUDP-356376 Allow Search to work with x509 cluster auth (#568)

# Summary
With the switch to gRPC for mongot in #527 it's now possible for Search
to be deployed against clusters that use x509 internal authentication.
This removes the validation check for the internal cluster
authentication mode and adds a new e2e test based on the existing
`search_enterprise_tls` test that verifies existing functionality
against a x509 cluster.

## Proof of Work
New test.

## Checklist

- [x] Have you linked a jira ticket and/or is the ticket in the title?
- [x] Have you checked whether your jira ticket required DOCSP changes?
- [x] Have you added changelog file?
    - use `skip-changelog` label if not needed
- refer to [Changelog files and Release
Notes](https://github.com/mongodb/mongodb-kubernetes/blob/master/CONTRIBUTING.md#changelog-files-and-release-notes)
section in CONTRIBUTING.md for more details

Author: fealebenpae

.evergreen-tasks.yml

@@ -1307,3 +1307,8 @@ tasks:
     tags: [ "patch-run" ]
     commands:
       - func: "e2e_test"
+
+  - name: e2e_search_enterprise_x509_cluster_auth
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"

.evergreen.yml

@@ -758,6 +758,7 @@ task_groups:
       # MongoDBSearch test group
       - e2e_search_enterprise_basic
       - e2e_search_enterprise_tls
+      - e2e_search_enterprise_x509_cluster_auth
     <<: *teardown_group
 
   # this task group contains just a one task, which is smoke testing whether the operator
@@ -1190,6 +1191,7 @@ task_groups:
     <<: *setup_and_teardown_task
     tasks:
       - e2e_search_enterprise_tls
+      - e2e_search_enterprise_x509_cluster_auth
     <<: *teardown_group
 
   # Tests features only supported on OM70 and OM80, its only upgrade test as we test upgrading from 6 to 7 or 7 to 8

changelog/20251103_feature_mongodbsearch_mongodb_deployments_using_x509.md

@@ -0,0 +1,7 @@
+---
+kind: feature
+date: 2025-11-03
+---
+
+* **MongoDBSearch**: MongoDB deployments using X509 internal cluster authentication are now supported. Previously MongoDB Search required SCRAM authentication among members of a MongoDB replica set. Note: SCRAM client authentication is still required, this change merely relaxes the requirements on internal cluster authentication.
+

controllers/searchcontroller/enterprise_search_source.go

@@ -81,9 +81,5 @@ func (r EnterpriseResourceSearchSource) Validate() error {
 		return xerrors.New("MongoDBSearch requires SCRAM authentication to be enabled")
 	}
 
-	if r.Spec.Security.GetInternalClusterAuthenticationMode() == util.X509 {
-		return xerrors.New("MongoDBSearch does not support X.509 internal cluster authentication")
-	}
-
 	return nil
 }

controllers/searchcontroller/enterprise_search_source_test.go

@@ -223,8 +223,7 @@ func TestEnterpriseResourceSearchSource_Validate(t *testing.T) {
 			resourceType:        mdbv1.ReplicaSet,
 			authModes:           []string{"SCRAM-SHA-256"},
 			internalClusterAuth: "X509",
-			expectError:         true,
-			expectedErrMsg:      "MongoDBSearch does not support X.509 internal cluster authentication",
+			expectError:         false,
 		},
 		{
 			name:                "Valid internal cluster auth - empty",

docker/mongodb-kubernetes-tests/tests/search/search_enterprise_x509_cluster_auth.py

@@ -0,0 +1,261 @@
+import yaml
+from kubetester import create_or_update_secret, run_periodically, try_load
+from kubetester.certs import (
+    create_agent_tls_certs,
+    create_tls_certs,
+    create_x509_mongodb_tls_certs,
+)
+from kubetester.kubetester import KubernetesTester
+from kubetester.kubetester import fixture as yaml_fixture
+from kubetester.mongodb import MongoDB
+from kubetester.mongodb_search import MongoDBSearch
+from kubetester.mongodb_user import MongoDBUser
+from kubetester.omtester import skip_if_cloud_manager
+from kubetester.phase import Phase
+from pytest import fixture, mark
+from tests import test_logger
+from tests.common.search import movies_search_helper
+from tests.common.search.search_tester import SearchTester
+from tests.conftest import get_default_operator, get_issuer_ca_filepath
+from tests.search.om_deployment import get_ops_manager
+
+logger = test_logger.get_test_logger(__name__)
+
+ADMIN_USER_NAME = "mdb-admin-user"
+ADMIN_USER_PASSWORD = f"{ADMIN_USER_NAME}-password"
+
+MONGOT_USER_NAME = "search-sync-source"
+MONGOT_USER_PASSWORD = f"{MONGOT_USER_NAME}-password"
+
+USER_NAME = "mdb-user"
+USER_PASSWORD = f"{USER_NAME}-password"
+
+MDB_RESOURCE_NAME = "mdb-ent-tls"
+
+# MongoDBSearch TLS configuration
+MDBS_TLS_SECRET_NAME = "mdbs-tls-secret"
+
+
+@fixture(scope="function")
+def mdb(namespace: str, issuer_ca_configmap: str) -> MongoDB:
+    resource = MongoDB.from_yaml(
+        yaml_fixture("enterprise-replicaset-sample-mflix.yaml"),
+        name=MDB_RESOURCE_NAME,
+        namespace=namespace,
+    )
+
+    if try_load(resource):
+        return resource
+
+    resource.configure(om=get_ops_manager(namespace), project_name=MDB_RESOURCE_NAME)
+    resource.configure_custom_tls(issuer_ca_configmap, "certs")
+    resource["spec"]["security"]["authentication"] = {
+        "enabled": True,
+        "modes": ["X509", "SCRAM"],
+        "agents": {"mode": "X509"},
+        "internalCluster": "X509",
+    }
+
+    return resource
+
+
+@fixture(scope="function")
+def mdbs(namespace: str) -> MongoDBSearch:
+    resource = MongoDBSearch.from_yaml(yaml_fixture("search-minimal.yaml"), namespace=namespace, name=MDB_RESOURCE_NAME)
+
+    if try_load(resource):
+        return resource
+
+    # Add TLS configuration to MongoDBSearch
+    if "spec" not in resource:
+        resource["spec"] = {}
+
+    resource["spec"]["security"] = {"tls": {"certificateKeySecretRef": {"name": MDBS_TLS_SECRET_NAME}}}
+
+    return resource
+
+
+@fixture(scope="function")
+def admin_user(namespace: str) -> MongoDBUser:
+    resource = MongoDBUser.from_yaml(
+        yaml_fixture("mongodbuser-mdb-admin.yaml"), namespace=namespace, name=ADMIN_USER_NAME
+    )
+
+    if try_load(resource):
+        return resource
+
+    resource["spec"]["mongodbResourceRef"]["name"] = MDB_RESOURCE_NAME
+    resource["spec"]["username"] = resource.name
+    resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password"
+
+    return resource
+
+
+@fixture(scope="function")
+def user(namespace: str) -> MongoDBUser:
+    resource = MongoDBUser.from_yaml(yaml_fixture("mongodbuser-mdb-user.yaml"), namespace=namespace, name=USER_NAME)
+
+    if try_load(resource):
+        return resource
+
+    resource["spec"]["mongodbResourceRef"]["name"] = MDB_RESOURCE_NAME
+    resource["spec"]["username"] = resource.name
+    resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password"
+
+    return resource
+
+
+@fixture(scope="function")
+def mongot_user(namespace: str, mdbs: MongoDBSearch) -> MongoDBUser:
+    resource = MongoDBUser.from_yaml(
+        yaml_fixture("mongodbuser-search-sync-source-user.yaml"),
+        namespace=namespace,
+        name=f"{mdbs.name}-{MONGOT_USER_NAME}",
+    )
+
+    if try_load(resource):
+        return resource
+
+    resource["spec"]["mongodbResourceRef"]["name"] = MDB_RESOURCE_NAME
+    resource["spec"]["username"] = MONGOT_USER_NAME
+    resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password"
+
+    return resource
+
+
+@mark.e2e_search_enterprise_x509_cluster_auth
+def test_install_operator(namespace: str, operator_installation_config: dict[str, str]):
+    operator = get_default_operator(namespace, operator_installation_config=operator_installation_config)
+    operator.assert_is_running()
+
+
+@mark.e2e_search_enterprise_x509_cluster_auth
+@skip_if_cloud_manager
+def test_create_ops_manager(namespace: str):
+    ops_manager = get_ops_manager(namespace)
+    ops_manager.update()
+    ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=1200)
+    ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600)
+
+
+@mark.e2e_search_enterprise_x509_cluster_auth
+def test_install_tls_secrets_and_configmaps(namespace: str, mdb: MongoDB, mdbs: MongoDBSearch, issuer: str):
+    create_agent_tls_certs(issuer, namespace, mdb.name, "certs")
+    create_x509_mongodb_tls_certs(issuer, namespace, mdb.name, f"certs-{mdb.name}-clusterfile")
+    create_x509_mongodb_tls_certs(issuer, namespace, mdb.name, f"certs-{mdb.name}-cert", mdb.get_members())
+
+    search_service_name = f"{mdbs.name}-search-svc"
+    create_tls_certs(
+        issuer,
+        namespace,
+        f"{mdbs.name}-search",
+        replicas=1,
+        service_name=search_service_name,
+        additional_domains=[f"{search_service_name}.{namespace}.svc.cluster.local"],
+        secret_name=MDBS_TLS_SECRET_NAME,
+    )
+
+
+@mark.e2e_search_enterprise_x509_cluster_auth
+def test_create_database_resource(mdb: MongoDB):
+    mdb.update()
+    mdb.assert_reaches_phase(Phase.Running, timeout=300)
+
+
+@mark.e2e_search_enterprise_x509_cluster_auth
+def test_create_users(
+    namespace: str, admin_user: MongoDBUser, user: MongoDBUser, mongot_user: MongoDBUser, mdb: MongoDB
+):
+    create_or_update_secret(
+        namespace, name=admin_user["spec"]["passwordSecretKeyRef"]["name"], data={"password": ADMIN_USER_PASSWORD}
+    )
+    admin_user.update()
+    admin_user.assert_reaches_phase(Phase.Updated, timeout=300)
+
+    create_or_update_secret(
+        namespace, name=user["spec"]["passwordSecretKeyRef"]["name"], data={"password": USER_PASSWORD}
+    )
+    user.update()
+    user.assert_reaches_phase(Phase.Updated, timeout=300)
+
+    create_or_update_secret(
+        namespace, name=mongot_user["spec"]["passwordSecretKeyRef"]["name"], data={"password": MONGOT_USER_PASSWORD}
+    )
+    mongot_user.update()
+    mongot_user.assert_reaches_phase(Phase.Updated, timeout=300)
+
+
+@mark.e2e_search_enterprise_x509_cluster_auth
+def test_create_search_resource(mdbs: MongoDBSearch):
+    mdbs.update()
+    mdbs.assert_reaches_phase(Phase.Running, timeout=300)
+
+
+# After picking up MongoDBSearch CR, MongoDB reconciler will add mongod parameters to each process.
+# Due to how MongoDB reconciler works (blocking on waiting for agents and not changing the status to pending)
+# the phase won't be updated to Pending and we need to wait by checking agents' status directly in OM.
+@mark.e2e_search_enterprise_x509_cluster_auth
+def test_wait_for_agents_ready(mdb: MongoDB):
+    mdb.get_om_tester().wait_agents_ready()
+    mdb.assert_reaches_phase(Phase.Running, timeout=300)
+
+
+@mark.e2e_search_enterprise_x509_cluster_auth
+def test_wait_for_mongod_parameters(mdb: MongoDB):
+    # After search CR is deployed, MongoDB controller will pick it up
+    # and start adding search-related parameters to the automation config.
+    def check_mongod_parameters():
+        parameters_are_set = True
+        pod_parameters = []
+        for idx in range(mdb.get_members()):
+            mongod_config = yaml.safe_load(
+                KubernetesTester.run_command_in_pod_container(
+                    f"{mdb.name}-{idx}", mdb.namespace, ["cat", "/data/automation-mongod.conf"]
+                )
+            )
+            set_parameter = mongod_config.get("setParameter", {})
+            parameters_are_set = parameters_are_set and (
+                "mongotHost" in set_parameter and "searchIndexManagementHostAndPort" in set_parameter
+            )
+            pod_parameters.append(f"pod {idx} setParameter: {set_parameter}")
+
+        return parameters_are_set, f'Not all pods have mongot parameters set:\n{"\n".join(pod_parameters)}'
+
+    run_periodically(check_mongod_parameters, timeout=600)
+
+
+@mark.e2e_search_enterprise_x509_cluster_auth
+def test_search_restore_sample_database(mdb: MongoDB):
+    get_admin_sample_movies_helper(mdb).restore_sample_database()
+
+
+@mark.e2e_search_enterprise_x509_cluster_auth
+def test_search_create_search_index(mdb: MongoDB):
+    get_user_sample_movies_helper(mdb).create_search_index()
+
+
+@mark.e2e_search_enterprise_x509_cluster_auth
+def test_search_assert_search_query(mdb: MongoDB):
+    get_user_sample_movies_helper(mdb).assert_search_query(retry_timeout=60)
+
+
+def get_connection_string(mdb: MongoDB, user_name: str, user_password: str) -> str:
+    return f"mongodb://{user_name}:{user_password}@{mdb.name}-0.{mdb.name}-svc.{mdb.namespace}.svc.cluster.local:27017/?replicaSet={mdb.name}"
+
+
+def get_admin_sample_movies_helper(mdb):
+    return movies_search_helper.SampleMoviesSearchHelper(
+        SearchTester(
+            get_connection_string(mdb, ADMIN_USER_NAME, ADMIN_USER_PASSWORD),
+            use_ssl=True,
+            ca_path=get_issuer_ca_filepath(),
+        )
+    )
+
+
+def get_user_sample_movies_helper(mdb):
+    return movies_search_helper.SampleMoviesSearchHelper(
+        SearchTester(
+            get_connection_string(mdb, USER_NAME, USER_PASSWORD), use_ssl=True, ca_path=get_issuer_ca_filepath()
+        )
+    )