CLOUDP-331841 - remove the agent matrix work (#267)

# Summary

-
[TD](https://docs.google.com/document/d/1zu95dakDbIXCncakW2FbPSayKqPYPc-ga6YSEMMzB1M/edit?tab=t.0)
## Database:
from:


```
+------------------------------------------------------------+
|                        Pod: rs1                            |
+------------------------------------------------------------+
| Container 1: mongodb-agent (operator suffixed)             |
|   - entry: agent-launcher.sh                               |
|   - livenessProbe                                          |
|   - readinessProbe                                         |
+------------------------------------------------------------+
| Container 2: mongodb-enterprise-database                   |
|   - mongod                                                 |
|   - mongosh                                                |
+------------------------------------------------------------+
```

to:
```
+------------------------------------------------------------+
|                        Pod: rs1                            |
+------------------------------------------------------------+
| Container 1: mongodb-agent (not operator suffixed)         |
|   - entry: agent-launcher-shim.sh                          |
|   - dummy-livenessProbe                                    |
|   - dummy-readinessProbe                                   |
+------------------------------------------------------------+
| Container 2: utilities-holder (name tbd)                   |
|   - readinessProbe                                         |
|   - livenessProbe                                          |
|   - agent-launcher.sh                                      |
+------------------------------------------------------------+
| Container 3: mongodb-enterprise-database                   |
|   - mongod                                                 |
|   - mongosh                                                |
+------------------------------------------------------------+
```


AppDB:

from:

```
+------------------------------------------------------------+
|                        Pod: rs1                            |
+------------------------------------------------------------+
| Container 1: mongodb-agent (operator suffixed)             |
|   - entry: agent/mongodb-agent                             |
|   - agent-launcher.sh                                      |
|   - livenessProbe                                          |
|   - readinessProbe                                         |
+------------------------------------------------------------+
| Container 2: mongodb-enterprise-database                   |
|   - mongod                                                 |
|   - mongosh                                                |
+------------------------------------------------------------+
```

to:

```
+------------------------------------------------------------+
|                        Pod: rs1                            |
+------------------------------------------------------------+
| Container 1: mongodb-agent (not operator suffixed)         |
|   - entry: /usr/local/bin/setup-agent-files.sh &&          |
|            agent/mongodb-agent                             |
|   - agent-launcher-shim.sh (unused)                        |
|   - dummy-livenessProbe                                    |
|   - dummy-readinessProbe                                   |
+------------------------------------------------------------+
| Container 2: utilities-holder (name tbd)                   |
|   - readinessProbe                                         |
|   - livenessProbe                                          |
+------------------------------------------------------------+
| Container 3: mongodb-enterprise-database                   |
|   - mongod                                                 |
|   - mongosh                                                |
+------------------------------------------------------------+
```


# Summary Changes
This pull request introduces significant changes to the MongoDB
Kubernetes operator, focusing on refactoring, architecture updates, and
bug fixes. The changes include removing unused dependencies, improving
the handling of static and non-static architectures, and enhancing test
coverage. Below is a categorized summary of the most important changes:

### Refactoring and Code Simplification
- No operator suffixed agent returned by the operator for the agent
- Consolidated logic for handling static and non-static architectures in
the `buildDatabaseStatefulSetConfigurationFunction` by introducing
shared and architecture-specific modifications.
[[1]](diffhunk://#diff-828cc93faac76a71ad9a81f7eb3aa6ab91159ab4941b21ee16e79004a1481359R491-R531)
[[2]](diffhunk://#diff-828cc93faac76a71ad9a81f7eb3aa6ab91159ab4941b21ee16e79004a1481359L669-R798)
- Renamed `InitDatabaseNonStaticImage` to `InitDatabaseImage` in
`DatabaseStatefulSetOptions` for clarity.

### Static and Non-Static Architecture Handling
- Introduced separate functions
(`buildStaticArchitecturePodTemplateSpec` and
`buildNonStaticArchitecturePodTemplateSpec`) to handle static and
non-static architectures, improving modularity and maintainability.
- Updated the `AppDbStatefulSet` function to include
`opts.InitAppDBImage` when defining init containers, ensuring proper
handling of both architectures.

### Enhancements to StatefulSet and Pod Templates
- Added a new container (`automationUtilitiesPodTemplateFunc`) to the
`appDbPodSpec` for managing utility scripts and hooks, along with
corresponding volume mounts.
[[1]](diffhunk://#diff-1af026339f8134b3e4d6303cb53503da7748cb30bcbb03f63d36d82f8aaf265aR125-R130)
[[2]](diffhunk://#diff-1af026339f8134b3e4d6303cb53503da7748cb30bcbb03f63d36d82f8aaf265aL133-R146)
- Adjusted the `AutomationAgentCommand` to include additional startup
parameters for better configuration.


## Proof of Work

- passing ci
https://spruce.mongodb.com/version/68947bdc57d3640007d48db0/tasks?page=0&sorts=STATUS%3AASC%3BBASE_STATUS%3ADESC

## Checklist
- [ ] Have you linked a jira ticket and/or is the ticket in the title?
- [ ] Have you checked whether your jira ticket required DOCSP changes?
- [ ] Have you checked for release_note changes?

Author: nammn

.evergreen.yml

@@ -462,9 +462,6 @@ tasks:
           skip_tags: ubuntu,release
 
   - name: build_agent_images_ubi
-    depends_on:
-      - name: build_init_database_image_ubi
-        variant: init_test_run
     commands:
       - func: clone
       - func: setup_building_host

changelog/20250806_fix_changing_container_setup_of_static_architecture.md

@@ -0,0 +1,8 @@
+---
+title: Changing container setup of static architecture
+kind: fix
+date: 2025-08-06
+---
+
+* This change fixes the current complex and difficult-to-maintain architecture for stateful set containers, which relies on an "agent matrix" to map operator and agent versions which led to a sheer amount of images.
+* We solve this by shifting to a 3-container setup. This new design eliminates the need for the operator-version/agent-version matrix by adding one additional container containing all required binaries. This architecture maps to what we already do with the mongodb-database container.

controllers/operator/common_controller.go

@@ -47,7 +47,6 @@ import (
 	"github.com/mongodb/mongodb-kubernetes/pkg/util/architectures"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util/env"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util/stringutil"
-	"github.com/mongodb/mongodb-kubernetes/pkg/util/versionutil"
 	"github.com/mongodb/mongodb-kubernetes/pkg/vault"
 )
 
@@ -684,9 +683,7 @@ func (r *ReconcileCommonController) getAgentVersion(conn om.Connection, omVersio
 		return "", err
 	} else {
 		log.Debugf("Using agent version %s", agentVersion)
-		currentOperatorVersion := versionutil.StaticContainersOperatorVersion()
-		log.Debugf("Using Operator version: %s", currentOperatorVersion)
-		return agentVersion + "_" + currentOperatorVersion, nil
+		return agentVersion, nil
 	}
 }
 

controllers/operator/construct/appdb_construction.go

@@ -122,6 +122,8 @@ func appDbPodSpec(initContainerImage string, om om.MongoDBOpsManager) podtemplat
 		construct.AgentName,
 		container.WithResourceRequirements(buildRequirementsFromPodSpec(*appdbPodSpec)),
 	)
+	scriptsVolumeMount := statefulset.CreateVolumeMount("agent-scripts", "/opt/scripts", statefulset.WithReadOnly(false))
+	hooksVolumeMount := statefulset.CreateVolumeMount("hooks", "/hooks", statefulset.WithReadOnly(false))
 
 	initUpdateFunc := podtemplatespec.NOOP()
 	if !architectures.IsRunningStaticArchitecture(om.Annotations) {
@@ -130,8 +132,6 @@ func appDbPodSpec(initContainerImage string, om om.MongoDBOpsManager) podtemplat
 		// volumes of different containers.
 		initUpdateFunc = func(templateSpec *corev1.PodTemplateSpec) {
 			templateSpec.Spec.InitContainers = []corev1.Container{}
-			scriptsVolumeMount := statefulset.CreateVolumeMount("agent-scripts", "/opt/scripts", statefulset.WithReadOnly(false))
-			hooksVolumeMount := statefulset.CreateVolumeMount("hooks", "/hooks", statefulset.WithReadOnly(false))
 			podtemplatespec.WithInitContainer(InitAppDbContainerName, buildAppDBInitContainer(initContainerImage, []corev1.VolumeMount{scriptsVolumeMount, hooksVolumeMount}))(templateSpec)
 		}
 	}
@@ -233,6 +233,12 @@ func CAConfigMapName(appDb om.AppDBSpec, log *zap.SugaredLogger) string {
 // and volumemounts for TLS.
 func tlsVolumes(appDb om.AppDBSpec, podVars *env.PodEnvVars, log *zap.SugaredLogger) podtemplatespec.Modification {
 	volumesToAdd, volumeMounts := getTLSVolumesAndVolumeMounts(appDb, podVars, log)
+
+	// Add agent API key volume mount if not using vault and monitoring is enabled
+	if !vault.IsVaultSecretBackend() && ShouldEnableMonitoring(podVars) {
+		volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(AgentAPIKeyVolumeName, AgentAPIKeySecretPath))
+	}
+
 	volumesFunc := func(spec *corev1.PodTemplateSpec) {
 		for _, v := range volumesToAdd {
 			podtemplatespec.WithVolume(v)(spec)
@@ -380,7 +386,7 @@ func AppDbStatefulSet(opsManager om.MongoDBOpsManager, podVars *env.PodEnvVars,
 	externalDomain := appDb.GetExternalDomainForMemberCluster(scaler.MemberClusterName())
 
 	if ShouldEnableMonitoring(podVars) {
-		monitoringModification = addMonitoringContainer(*appDb, *podVars, opts, externalDomain, log)
+		monitoringModification = addMonitoringContainer(*appDb, *podVars, opts, externalDomain, architectures.IsRunningStaticArchitecture(opsManager.Annotations), log)
 	} else {
 		// Otherwise, let's remove for now every podTemplateSpec related to monitoring
 		// We will apply them when enabling monitoring
@@ -390,7 +396,7 @@ func AppDbStatefulSet(opsManager om.MongoDBOpsManager, podVars *env.PodEnvVars,
 	}
 
 	// We copy the Automation Agent command from community and add the agent startup parameters
-	automationAgentCommand := construct.AutomationAgentCommand(true, opsManager.Spec.AppDB.GetAgentLogLevel(), opsManager.Spec.AppDB.GetAgentLogFile(), opsManager.Spec.AppDB.GetAgentMaxLogFileDurationHours())
+	automationAgentCommand := construct.AutomationAgentCommand(architectures.IsRunningStaticArchitecture(opsManager.Annotations), true, opsManager.Spec.AppDB.GetAgentLogLevel(), opsManager.Spec.AppDB.GetAgentLogFile(), opsManager.Spec.AppDB.GetAgentMaxLogFileDurationHours())
 	idx := len(automationAgentCommand) - 1
 	automationAgentCommand[idx] += appDb.AutomationAgent.StartupParameters.ToCommandLineArgs()
 
@@ -403,13 +409,10 @@ func AppDbStatefulSet(opsManager om.MongoDBOpsManager, podVars *env.PodEnvVars,
 		MountPath: "/var/lib/automation/config/acVersion",
 	}
 
-	// Here we ask to craete init containers which also creates required volumens.
+	// Here we ask to create init containers which also creates required volumes.
 	// Note that we provide empty images for init containers. They are not important
-	// at this stage beucase later we will define our own init containers for non-static architecture.
-	mod := construct.BuildMongoDBReplicaSetStatefulSetModificationFunction(&opsManager.Spec.AppDB, scaler, opts.MongodbImage, opts.AgentImage, "", "", true)
-	if architectures.IsRunningStaticArchitecture(opsManager.Annotations) {
-		mod = construct.BuildMongoDBReplicaSetStatefulSetModificationFunction(&opsManager.Spec.AppDB, scaler, opts.MongodbImage, opts.AgentImage, "", "", false)
-	}
+	// at this stage because later we will define our own init containers for non-static architecture.
+	mod := construct.BuildMongoDBReplicaSetStatefulSetModificationFunction(&opsManager.Spec.AppDB, scaler, opts.MongodbImage, opts.AgentImage, "", "", !architectures.IsRunningStaticArchitecture(opsManager.Annotations), opts.InitAppDBImage)
 
 	sts := statefulset.New(
 		mod,
@@ -493,7 +496,7 @@ func getVolumeMountIndexByName(mounts []corev1.VolumeMount, name string) int {
 // addMonitoringContainer returns a podtemplatespec modification that adds the monitoring container to the AppDB Statefulset.
 // Note that this replicates some code from the functions that do this for the base AppDB Statefulset. After many iterations
 // this was deemed to be an acceptable compromise to make code clearer and more maintainable.
-func addMonitoringContainer(appDB om.AppDBSpec, podVars env.PodEnvVars, opts AppDBStatefulSetOptions, externalDomain *string, log *zap.SugaredLogger) podtemplatespec.Modification {
+func addMonitoringContainer(appDB om.AppDBSpec, podVars env.PodEnvVars, opts AppDBStatefulSetOptions, externalDomain *string, isStatic bool, log *zap.SugaredLogger) podtemplatespec.Modification {
 	var monitoringAcVolume corev1.Volume
 	var monitoringACFunc podtemplatespec.Modification
 
@@ -516,7 +519,7 @@ func addMonitoringContainer(appDB om.AppDBSpec, podVars env.PodEnvVars, opts App
 	}
 	// Construct the command by concatenating:
 	// 1. The base command - from community
-	command := construct.MongodbUserCommandWithAPIKeyExport
+	command := construct.GetMongodbUserCommandWithAPIKeyExport(isStatic)
 	command += "agent/mongodb-agent"
 	command += " -healthCheckFilePath=" + monitoringAgentHealthStatusFilePathValue
 	command += " -serveStatusPort=5001"

controllers/operator/construct/construction_test.go

@@ -28,20 +28,20 @@ func TestBuildStatefulSet_PersistentFlagStatic(t *testing.T) {
 	mdb := mdbv1.NewReplicaSetBuilder().SetPersistent(nil).Build()
 	set := DatabaseStatefulSet(*mdb, ReplicaSetOptions(GetPodEnvOptions()), zap.S())
 	assert.Len(t, set.Spec.VolumeClaimTemplates, 1)
-	assert.Len(t, set.Spec.Template.Spec.Containers[0].VolumeMounts, 7)
+	assert.Len(t, set.Spec.Template.Spec.Containers[0].VolumeMounts, 8)
 	assert.Len(t, set.Spec.Template.Spec.Containers[1].VolumeMounts, 7)
 
 	mdb = mdbv1.NewReplicaSetBuilder().SetPersistent(util.BooleanRef(true)).Build()
 	set = DatabaseStatefulSet(*mdb, ReplicaSetOptions(GetPodEnvOptions()), zap.S())
 	assert.Len(t, set.Spec.VolumeClaimTemplates, 1)
-	assert.Len(t, set.Spec.Template.Spec.Containers[0].VolumeMounts, 7)
+	assert.Len(t, set.Spec.Template.Spec.Containers[0].VolumeMounts, 8)
 	assert.Len(t, set.Spec.Template.Spec.Containers[1].VolumeMounts, 7)
 
 	// If no persistence is set then we still mount init scripts
 	mdb = mdbv1.NewReplicaSetBuilder().SetPersistent(util.BooleanRef(false)).Build()
 	set = DatabaseStatefulSet(*mdb, ReplicaSetOptions(GetPodEnvOptions()), zap.S())
 	assert.Len(t, set.Spec.VolumeClaimTemplates, 0)
-	assert.Len(t, set.Spec.Template.Spec.Containers[0].VolumeMounts, 7)
+	assert.Len(t, set.Spec.Template.Spec.Containers[0].VolumeMounts, 8)
 	assert.Len(t, set.Spec.Template.Spec.Containers[1].VolumeMounts, 7)
 }
 
@@ -111,6 +111,7 @@ func TestBuildStatefulSet_PersistentVolumeClaimSingleStatic(t *testing.T) {
 		{Name: util.PvcNameData, MountPath: util.PvcMountPathData, SubPath: util.PvcNameData},
 		{Name: util.PvcNameData, MountPath: util.PvcMountPathJournal, SubPath: util.PvcNameJournal},
 		{Name: util.PvcNameData, MountPath: util.PvcMountPathLogs, SubPath: util.PvcNameLogs},
+		{Name: PvcNameDatabaseScripts, MountPath: PvcMountPathScripts, ReadOnly: false},
 	})
 }