CLOUDP-321076: expose prometheus metrics settings in MongoDBSearch

api/v1/search/mongodbsearch_types.go

@@ -17,11 +17,11 @@ import (
 )
 
 const (
-	MongotDefaultWireprotoPort      = 27027
-	MongotDefaultGrpcPort           = 27028
-	MongotDefaultMetricsPort        = 9946
-	MongotDefautHealthCheckPort     = 8080
-	MongotDefaultSyncSourceUsername = "search-sync-source"
+	MongotDefaultWireprotoPort      int32 = 27027
+	MongotDefaultGrpcPort           int32 = 27028
+	MongotDefaultPrometheusPort     int32 = 9946
+	MongotDefautHealthCheckPort     int32 = 8080
+	MongotDefaultSyncSourceUsername       = "search-sync-source"
 
 	ForceWireprotoAnnotation = "mongodb.com/v1.force-search-wireproto"
 )
@@ -30,6 +30,23 @@ func init() {
 	v1.SchemeBuilder.Register(&MongoDBSearch{}, &MongoDBSearchList{})
 }
 
+type Prometheus struct {
+	// Port where metrics endpoint will be exposed on. Defaults to 9946.
+	// +optional
+	// +kubebuilder:validation:Minimum=0
+	// +kubebuilder:validation:Maximum=65535
+	Port int `json:"port,omitempty"`
+}
+
+func (p *Prometheus) GetPort() int32 {
+	if p.Port == 0 {
+		return MongotDefaultPrometheusPort
+	}
+
+	//nolint:gosec
+	return int32(p.Port)
+}
+
 type MongoDBSearchSpec struct {
 	// Optional version of MongoDB Search component (mongot). If not set, then the operator will set the most appropriate version of MongoDB Search.
 	// +optional
@@ -54,6 +71,9 @@ type MongoDBSearchSpec struct {
 	// +kubebuilder:validation:Enum=TRACE;DEBUG;INFO;WARN;ERROR
 	// +optional
 	LogLevel mdb.LogLevel `json:"logLevel,omitempty"`
+	// Configure prometheus metrics endpoint in mongot. If not set, the metrics endpoint will be disabled.
+	// +optional
+	Prometheus *Prometheus `json:"prometheus,omitempty"`
 }
 
 type MongoDBSource struct {
@@ -218,10 +238,6 @@ func (s *MongoDBSearch) GetMongotGrpcPort() int32 {
 	return MongotDefaultGrpcPort
 }
 
-func (s *MongoDBSearch) GetMongotMetricsPort() int32 {
-	return MongotDefaultMetricsPort
-}
-
 // TLSSecretNamespacedName will get the namespaced name of the Secret containing the server certificate and key
 func (s *MongoDBSearch) TLSSecretNamespacedName() types.NamespacedName {
 	return types.NamespacedName{Name: s.Spec.Security.TLS.CertificateKeySecret.Name, Namespace: s.Namespace}
@@ -263,3 +279,7 @@ func (s *MongoDBSearch) GetEffectiveMongotPort() int32 {
 	}
 	return s.GetMongotGrpcPort()
 }
+
+func (s *MongoDBSearch) GetPrometheus() *Prometheus {
+	return s.Spec.Prometheus
+}

changelog/20251030_feature_update_mongodb_search_2nd_preview.md

@@ -0,0 +1,14 @@
+---
+kind: feature
+date: 2025-10-30
+---
+
+* **MongoDBSearch**:
+  * Switched to gRPC and mTLS for internal communication between mongod and mongot.
+    * Since MCK 1.4 the `mongod` and `mongot` processess communicated using the MongoDB Wire Protocol and used keyfile authentication. This release switches that to gRPC with mTLS authentication. gRPC will allow for load-balancing search queries against multiple `mongot` processes in the future, and mTLS decouples the internal cluster authentication mode and credentials among `mongod` processes from the connection to the `mongot` process. The Operator will automatically enable gRPC for existing and new workloads, and will enable mTLS authentication if both Database Server and `MongoDBSearch` resource are configured for TLS.
+  * Exposed configuration settings for mongot's prometheus metrics endpoint.
+    * By default, if `spec.prometheus` field is not provided then metrics endpoint in mongot is disabled. **This is a breaking change**. Previously the metrics endpoing was always enabled on port 9946.
+    * To enable prometheus metrics endpoint specify empty `spec.prometheus:` field. It will enable metrics endpoint on a default port (9946). To change the port, set it in `spec.prometheus.port` field.
+  * Simplified MongoDB Search setup: Removed the custom Search Coordinator polyfill (a piece of compatibility code previously needed to add the required permissions), as MongoDB 8.2.0 and later now include the necessary permissions via the built-in searchCoordinator role.
+  * Updated the default `mongodb/mongodb-search` image version to 0.55.0. This is the version MCK uses if `.spec.version` is not specified.
+  * MongoDB deployments using X509 internal cluster authentication are now supported. Previously MongoDB Search required SCRAM authentication among members of a MongoDB replica set. Note: SCRAM client authentication is still required, this change merely relaxes the requirements on internal cluster authentication.

changelog/20251106_feature_mongodbsearch_updated_the_default_mongodbmongodb.md

@@ -2,5 +2,3 @@
 kind: feature
 date: 2025-11-06
 ---
-
-* **MongoDBSearch**: Updated the default `mongodb/mongodb-search` image version to 0.55.0. This is the version MCK uses if `.spec.version` is not specified.

config/crd/bases/mongodb.com_mongodbsearch.yaml

@@ -106,6 +106,17 @@ spec:
                         type: string
                     type: object
                 type: object
+              prometheus:
+                description: Configure prometheus metrics endpoint in mongot. If not
+                  set, the metrics endpoint will be disabled.
+                properties:
+                  port:
+                    description: Port where metrics endpoint will be exposed on. Defaults
+                      to 9946.
+                    maximum: 65535
+                    minimum: 0
+                    type: integer
+                type: object
               resourceRequirements:
                 description: Configure resource requests and limits for the MongoDB
                   Search pods.

controllers/operator/appdbreplicaset_controller_test.go

@@ -3,10 +3,10 @@ package operator
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 	"time"
 
@@ -54,15 +54,14 @@ func init() {
 
 // getReleaseJsonPath searches for a specified target directory by traversing the directory tree backwards from the current working directory
 func getReleaseJsonPath() (string, error) {
-	repositoryRootDirName := "mongodb-kubernetes"
 	releaseFileName := "release.json"
 
 	currentDir, err := os.Getwd()
 	if err != nil {
 		return "", err
 	}
 	for currentDir != "/" {
-		if strings.HasSuffix(currentDir, repositoryRootDirName) {
+		if _, err := os.Stat(filepath.Join(currentDir, releaseFileName)); !errors.Is(err, os.ErrNotExist) {
 			return filepath.Join(currentDir, releaseFileName), nil
 		}
 		currentDir = filepath.Dir(currentDir)

controllers/operator/mongodbsearch_controller_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/ghodss/yaml"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -22,12 +23,12 @@ import (
 	"github.com/mongodb/mongodb-kubernetes/api/v1/status"
 	userv1 "github.com/mongodb/mongodb-kubernetes/api/v1/user"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/mock"
-	"github.com/mongodb/mongodb-kubernetes/controllers/operator/workflow"
 	"github.com/mongodb/mongodb-kubernetes/controllers/searchcontroller"
 	mdbcv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1"
 	"github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1/common"
 	"github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/mongot"
 	"github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/util/constants"
+	"github.com/mongodb/mongodb-kubernetes/pkg/util"
 )
 
 func newMongoDBCommunity(name, namespace string) *mdbcv1.MongoDBCommunity {
@@ -135,10 +136,6 @@ func buildExpectedMongotConfig(search *searchv1.MongoDBSearch, mdbc *mdbcv1.Mong
 			},
 			Wireproto: wireprotoServer,
 		},
-		Metrics: mongot.ConfigMetrics{
-			Enabled: true,
-			Address: fmt.Sprintf("0.0.0.0:%d", search.GetMongotMetricsPort()),
-		},
 		HealthCheck: mongot.ConfigHealthCheck{
 			Address: fmt.Sprintf("0.0.0.0:%d", search.GetMongotHealthCheckPort()),
 		},
@@ -205,22 +202,16 @@ func TestMongoDBSearchReconcile_Success(t *testing.T) {
 			mdbc := newMongoDBCommunity("mdb", mock.TestNamespace)
 			reconciler, c := newSearchReconciler(mdbc, search)
 
-			res, err := reconciler.Reconcile(
-				ctx,
-				reconcile.Request{NamespacedName: types.NamespacedName{Name: search.Name, Namespace: search.Namespace}},
-			)
-			expected, _ := workflow.OK().ReconcileResult()
-			assert.NoError(t, err)
-			assert.Equal(t, expected, res)
+			checkSearchReconcileSuccessful(ctx, t, reconciler, c, search)
 
 			svc := &corev1.Service{}
-			err = c.Get(ctx, search.SearchServiceNamespacedName(), svc)
+			err := c.Get(ctx, search.SearchServiceNamespacedName(), svc)
 			assert.NoError(t, err)
 			servicePortNames := []string{}
 			for _, port := range svc.Spec.Ports {
 				servicePortNames = append(servicePortNames, port.Name)
 			}
-			expectedPortNames := []string{"mongot-grpc", "metrics", "healthcheck"}
+			expectedPortNames := []string{"mongot-grpc", "healthcheck"}
 			if tc.withWireproto {
 				expectedPortNames = append(expectedPortNames, "mongot-wireproto")
 			}
@@ -254,14 +245,42 @@ func checkSearchReconcileFailed(
 		reconcile.Request{NamespacedName: types.NamespacedName{Name: search.Name, Namespace: search.Namespace}},
 	)
 	assert.NoError(t, err)
-	assert.True(t, res.RequeueAfter > 0)
+	assert.Less(t, res.RequeueAfter, util.TWENTY_FOUR_HOURS)
 
 	updated := &searchv1.MongoDBSearch{}
 	assert.NoError(t, c.Get(ctx, types.NamespacedName{Name: search.Name, Namespace: search.Namespace}, updated))
 	assert.Equal(t, status.PhaseFailed, updated.Status.Phase)
 	assert.Contains(t, updated.Status.Message, expectedMsg)
 }
 
+// checkSearchReconcileSuccessful performs reconcile to check if it gets to a Running state.
+// In case it's a first reconcile and still Pending it's retried with mocked sts simulated as ready.
+func checkSearchReconcileSuccessful(
+	ctx context.Context,
+	t *testing.T,
+	reconciler *MongoDBSearchReconciler,
+	c client.Client,
+	search *searchv1.MongoDBSearch,
+) {
+	namespacedName := types.NamespacedName{Name: search.Name, Namespace: search.Namespace}
+	res, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: namespacedName})
+	require.NoError(t, err)
+	mdbs := &searchv1.MongoDBSearch{}
+	require.NoError(t, c.Get(ctx, namespacedName, mdbs))
+	if mdbs.Status.Phase == status.PhasePending {
+		// mark mocked search statefulset as ready to not return Pending this time
+		require.NoError(t, mock.MarkAllStatefulSetsAsReady(ctx, search.Namespace, c))
+
+		res, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: namespacedName})
+		require.NoError(t, err)
+		mdbs = &searchv1.MongoDBSearch{}
+		require.NoError(t, c.Get(ctx, namespacedName, mdbs))
+	}
+
+	require.Equal(t, util.TWENTY_FOUR_HOURS, res.RequeueAfter)
+	require.Equal(t, status.PhaseRunning, mdbs.Status.Phase)
+}
+
 func TestMongoDBSearchReconcile_InvalidVersion(t *testing.T) {
 	ctx := context.Background()
 	search := newMongoDBSearch("search", mock.TestNamespace, "mdb")

controllers/searchcontroller/mongodbsearch_reconcile_helper.go

@@ -76,7 +76,7 @@ func (r *MongoDBSearchReconcileHelper) Reconcile(ctx context.Context, log *zap.S
 	if _, err := commoncontroller.UpdateStatus(ctx, r.client, r.mdbSearch, workflowStatus, log); err != nil {
 		return workflow.Failed(err)
 	}
-	return workflow.OK()
+	return workflowStatus
 }
 
 func (r *MongoDBSearchReconcileHelper) reconcile(ctx context.Context, log *zap.SugaredLogger) workflow.Status {
@@ -312,7 +312,7 @@ func buildSearchHeadlessService(search *searchv1.MongoDBSearch) corev1.Service {
 		SetLabels(labels).
 		SetServiceType(corev1.ServiceTypeClusterIP).
 		SetClusterIP("None").
-		SetPublishNotReadyAddresses(true).
+		SetPublishNotReadyAddresses(false).
 		SetOwnerReferences(search.GetOwnerReferences())
 
 	if search.IsWireprotoEnabled() {
@@ -331,12 +331,14 @@ func buildSearchHeadlessService(search *searchv1.MongoDBSearch) corev1.Service {
 		TargetPort: intstr.FromInt32(search.GetMongotGrpcPort()),
 	})
 
-	serviceBuilder.AddPort(&corev1.ServicePort{
-		Name:       "metrics",
-		Protocol:   corev1.ProtocolTCP,
-		Port:       search.GetMongotMetricsPort(),
-		TargetPort: intstr.FromInt32(search.GetMongotMetricsPort()),
-	})
+	if prometheus := search.GetPrometheus(); prometheus != nil {
+		serviceBuilder.AddPort(&corev1.ServicePort{
+			Name:       "prometheus",
+			Protocol:   corev1.ProtocolTCP,
+			Port:       prometheus.GetPort(),
+			TargetPort: intstr.FromInt32(prometheus.GetPort()),
+		})
+	}
 
 	serviceBuilder.AddPort(&corev1.ServicePort{
 		Name:       "healthcheck",
@@ -385,10 +387,14 @@ func createMongotConfig(search *searchv1.MongoDBSearch, db SearchSourceDBResourc
 				},
 			}
 		}
-		config.Metrics = mongot.ConfigMetrics{
-			Enabled: true,
-			Address: fmt.Sprintf("0.0.0.0:%d", search.GetMongotMetricsPort()),
+
+		if prometheus := search.GetPrometheus(); prometheus != nil {
+			config.Metrics = mongot.ConfigMetrics{
+				Enabled: true,
+				Address: fmt.Sprintf("0.0.0.0:%d", prometheus.GetPort()),
+			}
 		}
+
 		config.HealthCheck = mongot.ConfigHealthCheck{
 			Address: fmt.Sprintf("0.0.0.0:%d", search.GetMongotHealthCheckPort()),
 		}
@@ -473,9 +479,9 @@ func (r *MongoDBSearchReconcileHelper) getMongotImage() string {
 		return ""
 	}
 
-	for _, container := range r.mdbSearch.Spec.StatefulSetConfiguration.SpecWrapper.Spec.Template.Spec.Containers {
-		if container.Name == MongotContainerName {
-			return container.Image
+	for _, c := range r.mdbSearch.Spec.StatefulSetConfiguration.SpecWrapper.Spec.Template.Spec.Containers {
+		if c.Name == MongotContainerName {
+			return c.Image
 		}
 	}