CLOUDP-338399: External mongod for Search (#308)

# Summary

This pull request introduces support for using external MongoDB
deployments as sources for `MongoDBSearch` resources, in addition to the
previously supported operator-managed deployments. The changes include
updates to the CRD API, controller logic, and supporting code to handle
configuration, validation, and reconciliation for external sources.
Additionally, new end-to-end (e2e) test tasks are added to ensure this
functionality is covered.

**External MongoDB Source Support**

* Added new fields to the `MongoDBSource` spec (`ExternalMongoDBSource`
and supporting types) to allow specifying external MongoDB deployments,
including host/port, credentials, and TLS settings.
* Added logic to distinguish between operator-managed and external
sources throughout the controller and resource handling code, including
the `IsExternalMongoDBSource` method and updates to resource reference
logic.

**Controller and Reconciliation Logic Updates**

* Refactored the `SearchSourceDBResource` interface and its
implementations to support both internal and external MongoDB sources,
including connection details and version validation.
* Updated reconciliation logic to handle external sources, including
skipping version validation and resource watching when not applicable.
* Adjusted service and StatefulSet construction to use new abstraction
for host seeds and service accounts.

**Testing and CI**

* Added new e2e test tasks for external MongoDB source scenarios
(`e2e_search_external_basic` and `e2e_search_external_tls`) and included
them in the appropriate task groups to ensure coverage in CI.
* Updated tests and validation helpers to use the new interface methods
and logic.

These changes collectively enable `MongoDBSearch` resources to work with
both operator-managed and external MongoDB deployments, increasing
flexibility for users and improving the robustness of the operator.

## Proof of Work

Tests pass

Author: anandsyncs

.evergreen-tasks.yml

@@ -1290,3 +1290,13 @@ tasks:
     tags: ["patch-run"]
     commands:
       - func: "e2e_test"
+
+  - name: e2e_search_external_basic
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
+  - name: e2e_search_external_tls
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"

.evergreen.yml

@@ -646,6 +646,8 @@ task_groups:
       - e2e_community_replicaset_scale
       - e2e_search_community_basic
       - e2e_search_community_tls
+      - e2e_search_external_basic
+      - e2e_search_external_tls
 
   # This is the task group that contains all the tests run in the e2e_mdb_kind_ubuntu_cloudqa build variant
   - name: e2e_mdb_kind_cloudqa_task_group

api/v1/search/mongodbsearch_types.go

@@ -52,11 +52,26 @@ type MongoDBSource struct {
 	// +optional
 	MongoDBResourceRef *userv1.MongoDBResourceRef `json:"mongodbResourceRef,omitempty"`
 	// +optional
+	ExternalMongoDBSource *ExternalMongoDBSource `json:"external,omitempty"`
+	// +optional
 	PasswordSecretRef *userv1.SecretKeyRef `json:"passwordSecretRef,omitempty"`
 	// +optional
 	Username *string `json:"username,omitempty"`
 }
 
+type ExternalMongoDBSource struct {
+	HostAndPorts        []string             `json:"hostAndPorts,omitempty"`
+	KeyFileSecretKeyRef *userv1.SecretKeyRef `json:"keyFileSecretRef,omitempty"` // This is the mongod credential used to connect to the external MongoDB deployment
+	// +optional
+	TLS *ExternalMongodTLS `json:"tls,omitempty"` // TLS configuration for the external MongoDB deployment
+}
+
+type ExternalMongodTLS struct {
+	Enabled bool `json:"enabled"`
+	// +optional
+	CA *corev1.LocalObjectReference `json:"ca,omitempty"`
+}
+
 type Security struct {
 	// +optional
 	TLS TLS `json:"tls"`
@@ -170,13 +185,17 @@ func (s *MongoDBSearch) GetOwnerReferences() []metav1.OwnerReference {
 	return []metav1.OwnerReference{ownerReference}
 }
 
-func (s *MongoDBSearch) GetMongoDBResourceRef() userv1.MongoDBResourceRef {
+func (s *MongoDBSearch) GetMongoDBResourceRef() *userv1.MongoDBResourceRef {
+	if s.IsExternalMongoDBSource() {
+		return nil
+	}
+
 	mdbResourceRef := userv1.MongoDBResourceRef{Namespace: s.Namespace, Name: s.Name}
 	if s.Spec.Source != nil && s.Spec.Source.MongoDBResourceRef != nil && s.Spec.Source.MongoDBResourceRef.Name != "" {
 		mdbResourceRef.Name = s.Spec.Source.MongoDBResourceRef.Name
 	}
 
-	return mdbResourceRef
+	return &mdbResourceRef
 }
 
 func (s *MongoDBSearch) GetMongotPort() int32 {
@@ -201,3 +220,7 @@ func (s *MongoDBSearch) TLSOperatorSecretNamespacedName() types.NamespacedName {
 func (s *MongoDBSearch) GetMongotHealthCheckPort() int32 {
 	return MongotDefautHealthCheckPort
 }
+
+func (s *MongoDBSearch) IsExternalMongoDBSource() bool {
+	return s.Spec.Source != nil && s.Spec.Source.ExternalMongoDBSource != nil
+}

api/v1/search/zz_generated.deepcopy.go

@@ -182,6 +182,51 @@ spec:
                 type: object
               source:
                 properties:
+                  external:
+                    properties:
+                      hostAndPorts:
+                        items:
+                          type: string
+                        type: array
+                      keyFileSecretRef:
+                        description: |-
+                          SecretKeyRef is a reference to a value in a given secret in the same
+                          namespace. Based on:
+                          https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core
+                        properties:
+                          key:
+                            type: string
+                          name:
+                            type: string
+                        required:
+                          - name
+                        type: object
+                      tls:
+                        properties:
+                          ca:
+                            description: |-
+                              LocalObjectReference contains enough information to let you locate the
+                              referenced object inside the same namespace.
+                            properties:
+                              name:
+                                default: ""
+                                description: |-
+                                  Name of the referent.
+                                  This field is effectively required, but due to backwards compatibility is
+                                  allowed to be empty. Instances of this type with an empty value here are
+                                  almost certainly wrong.
+                                  TODO: Add other useful fields. apiVersion, kind, uid?
+                                  More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                  TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
+                                type: string
+                            type: object
+                            x-kubernetes-map-type: atomic
+                          enabled:
+                            type: boolean
+                        required:
+                          - enabled
+                        type: object
+                    type: object
                   mongodbResourceRef:
                     properties:
                       name:
@@ -209,8 +254,8 @@ spec:
                 type: object
               statefulSet:
                 description: |-
-                  StatefulSetConfiguration holds the optional custom StatefulSet
-                  that should be merged into the operator created one.
+                  StatefulSetSpec which the operator will apply to the MongoDB Search StatefulSet at the end of the reconcile loop. Use to provide necessary customizations,
+                  which aren't exposed as fields in the MongoDBSearch.spec.
                 properties:
                   metadata:
                     description: StatefulSetMetadataWrapper is a wrapper around Labels

config/crd/bases/mongodb.com_mongodbsearch.yaml

@@ -51,31 +51,46 @@ func (r *MongoDBSearchReconciler) Reconcile(ctx context.Context, request reconci
 		return result, err
 	}
 
-	sourceResource, err := getSourceMongoDBForSearch(ctx, r.kubeClient, mdbSearch)
+	sourceResource, mdbc, err := getSourceMongoDBForSearch(ctx, r.kubeClient, mdbSearch)
 	if err != nil {
 		return reconcile.Result{RequeueAfter: time.Second * util.RetryTimeSec}, err
 	}
 
-	r.mdbcWatcher.Watch(ctx, sourceResource.NamespacedName(), request.NamespacedName)
+	if mdbc != nil {
+		r.mdbcWatcher.Watch(ctx, mdbc.NamespacedName(), request.NamespacedName)
+	}
 
 	reconcileHelper := search_controller.NewMongoDBSearchReconcileHelper(kubernetesClient.NewClient(r.kubeClient), mdbSearch, sourceResource, r.operatorSearchConfig)
 
 	return reconcileHelper.Reconcile(ctx, log).ReconcileResult()
 }
 
-func getSourceMongoDBForSearch(ctx context.Context, kubeClient client.Client, search *searchv1.MongoDBSearch) (search_controller.SearchSourceDBResource, error) {
+func getSourceMongoDBForSearch(ctx context.Context, kubeClient client.Client, search *searchv1.MongoDBSearch) (search_controller.SearchSourceDBResource, *mdbcv1.MongoDBCommunity, error) {
+	if search.IsExternalMongoDBSource() {
+		return search_controller.NewSearchSourceDBResourceFromExternal(search.Namespace, search.Spec.Source.ExternalMongoDBSource), nil, nil
+	}
+
 	sourceMongoDBResourceRef := search.GetMongoDBResourceRef()
+	if sourceMongoDBResourceRef == nil {
+		return nil, nil, xerrors.New("MongoDBSearch source MongoDB resource reference is not set")
+	}
+
 	mdbcName := types.NamespacedName{Namespace: search.GetNamespace(), Name: sourceMongoDBResourceRef.Name}
 	mdbc := &mdbcv1.MongoDBCommunity{}
 	if err := kubeClient.Get(ctx, mdbcName, mdbc); err != nil {
-		return nil, xerrors.Errorf("error getting MongoDBCommunity %s: %w", mdbcName, err)
+		return nil, nil, xerrors.Errorf("error getting MongoDBCommunity %s: %w", mdbcName, err)
 	}
-	return search_controller.NewSearchSourceDBResourceFromMongoDBCommunity(mdbc), nil
+	return search_controller.NewSearchSourceDBResourceFromMongoDBCommunity(mdbc), mdbc, nil
 }
 
 func mdbcSearchIndexBuilder(rawObj client.Object) []string {
 	mdbSearch := rawObj.(*searchv1.MongoDBSearch)
-	return []string{mdbSearch.GetMongoDBResourceRef().Namespace + "/" + mdbSearch.GetMongoDBResourceRef().Name}
+	resourceRef := mdbSearch.GetMongoDBResourceRef()
+	if resourceRef == nil {
+		return []string{}
+	}
+
+	return []string{resourceRef.Namespace + "/" + resourceRef.Name}
 }
 
 func AddMongoDBSearchController(ctx context.Context, mgr manager.Manager, operatorSearchConfig search_controller.OperatorSearchConfig) error {

controllers/operator/mongodbsearch_controller.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/blang/semver"
 	"github.com/ghodss/yaml"
 	"go.uber.org/zap"
 	"golang.org/x/xerrors"
@@ -82,7 +81,7 @@ func (r *MongoDBSearchReconcileHelper) reconcile(ctx context.Context, log *zap.S
 	log = log.With("MongoDBSearch", r.mdbSearch.NamespacedName())
 	log.Infof("Reconciling MongoDBSearch")
 
-	if err := ValidateSearchSource(r.db); err != nil {
+	if err := r.db.ValidateMongoDBVersion(); err != nil {
 		return workflow.Failed(err)
 	}
 
@@ -123,7 +122,7 @@ func (r *MongoDBSearchReconcileHelper) reconcile(ctx context.Context, log *zap.S
 		return workflow.Failed(err)
 	}
 
-	if statefulSetStatus := statefulset.GetStatefulSetStatus(ctx, r.db.NamespacedName().Namespace, r.mdbSearch.StatefulSetNamespacedName().Name, r.client); !statefulSetStatus.IsOK() {
+	if statefulSetStatus := statefulset.GetStatefulSetStatus(ctx, r.mdbSearch.Namespace, r.mdbSearch.StatefulSetNamespacedName().Name, r.client); !statefulSetStatus.IsOK() {
 		return statefulSetStatus
 	}
 
@@ -334,10 +333,7 @@ func buildSearchHeadlessService(search *searchv1.MongoDBSearch) corev1.Service {
 
 func createMongotConfig(search *searchv1.MongoDBSearch, db SearchSourceDBResource) mongot.Modification {
 	return func(config *mongot.Config) {
-		var hostAndPorts []string
-		for i := range db.Members() {
-			hostAndPorts = append(hostAndPorts, fmt.Sprintf("%s-%d.%s.%s.svc.cluster.local:%d", db.Name(), i, db.DatabaseServiceName(), db.GetNamespace(), db.DatabasePort()))
-		}
+		hostAndPorts := db.HostSeeds()
 
 		config.SyncSource = mongot.ConfigSyncSource{
 			ReplicaSet: mongot.ConfigReplicaSet{
@@ -395,23 +391,17 @@ func mongotHostAndPort(search *searchv1.MongoDBSearch) string {
 	return fmt.Sprintf("%s.%s.svc.cluster.local:%d", svcName.Name, svcName.Namespace, search.GetMongotPort())
 }
 
-func ValidateSearchSource(db SearchSourceDBResource) error {
-	version, err := semver.ParseTolerant(db.GetMongoDBVersion())
-	if err != nil {
-		return xerrors.Errorf("error parsing MongoDB version '%s': %w", db.GetMongoDBVersion(), err)
-	} else if version.LT(semver.MustParse("8.0.10")) {
-		return xerrors.New("MongoDB version must be 8.0.10 or higher")
+func (r *MongoDBSearchReconcileHelper) ValidateSingleMongoDBSearchForSearchSource(ctx context.Context) error {
+	if r.mdbSearch.Spec.Source != nil && r.mdbSearch.Spec.Source.ExternalMongoDBSource != nil {
+		return nil
 	}
 
-	return nil
-}
-
-func (r *MongoDBSearchReconcileHelper) ValidateSingleMongoDBSearchForSearchSource(ctx context.Context) error {
+	ref := r.mdbSearch.GetMongoDBResourceRef()
 	searchList := &searchv1.MongoDBSearchList{}
 	if err := r.client.List(ctx, searchList, &client.ListOptions{
-		FieldSelector: fields.OneTermEqualSelector(MongoDBSearchIndexFieldName, r.db.GetNamespace()+"/"+r.db.Name()),
+		FieldSelector: fields.OneTermEqualSelector(MongoDBSearchIndexFieldName, ref.Namespace+"/"+ref.Name),
 	}); err != nil {
-		return xerrors.Errorf("Error listing MongoDBSearch resources for search source '%s': %w", r.db.Name(), err)
+		return xerrors.Errorf("Error listing MongoDBSearch resources for search source '%s': %w", ref.Name, err)
 	}
 
 	if len(searchList.Items) > 1 {
@@ -420,7 +410,7 @@ func (r *MongoDBSearchReconcileHelper) ValidateSingleMongoDBSearchForSearchSourc
 			resourceNames[i] = search.Name
 		}
 		return xerrors.Errorf(
-			"Found multiple MongoDBSearch resources for search source '%s': %s", r.db.Name(),
+			"Found multiple MongoDBSearch resources for search source '%s': %s", ref.Name,
 			strings.Join(resourceNames, ", "),
 		)
 	}

controllers/search_controller/mongodbsearch_reconcile_helper.go

@@ -64,7 +64,7 @@ func TestMongoDBSearchReconcileHelper_ValidateSearchSource(t *testing.T) {
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
 			db := NewSearchSourceDBResourceFromMongoDBCommunity(&c.mdbc)
-			err := ValidateSearchSource(db)
+			err := db.ValidateMongoDBVersion()
 			if c.expectedError == "" {
 				assert.NoError(t, err)
 			} else {