CLOUDP-328217: Automation agent password secret

.evergreen-tasks.yml

@@ -690,6 +690,46 @@ tasks:
     commands:
       - func: "e2e_test"
 
+  - name: e2e_sharded_cluster_scram_sha_256_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
+  - name: e2e_sharded_cluster_scram_sha_1_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
+  - name: e2e_sharded_cluster_x509_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
+  - name: e2e_replica_set_scram_sha_256_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
+  - name: e2e_replica_set_scram_sha_1_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
+  - name: e2e_replica_set_x509_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
+  - name: e2e_replica_set_ldap_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
+  - name: e2e_sharded_cluster_ldap_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
   # TODO: not used in any variant
   - name: e2e_replica_set_scram_x509_internal_cluster
     tags: [ "patch-run" ]

.evergreen.yml

@@ -713,6 +713,14 @@ task_groups:
       - e2e_sharded_cluster_scram_sha_1_user_connectivity
       - e2e_sharded_cluster_scram_x509_ic_manual_certs
       - e2e_sharded_cluster_external_access
+      - e2e_sharded_cluster_scram_sha_256_switch_project
+      - e2e_sharded_cluster_scram_sha_1_switch_project
+      - e2e_sharded_cluster_x509_switch_project
+      - e2e_replica_set_scram_sha_256_switch_project
+      - e2e_replica_set_scram_sha_1_switch_project
+      - e2e_replica_set_x509_switch_project
+      - e2e_replica_set_ldap_switch_project
+      - e2e_sharded_cluster_ldap_switch_project
       # e2e_auth_transitions_task_group
       - e2e_replica_set_scram_sha_and_x509
       - e2e_replica_set_x509_to_scram_transition

controllers/om/automation_config.go

@@ -1,19 +1,29 @@
 package om
 
 import (
+	"context"
 	"encoding/json"
+	"fmt"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/spf13/cast"
 	"k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/ldap"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/oidc"
+	"github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/secret"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util/generate"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util/maputil"
 )
 
+// The constants for the authentication secret
+const (
+	autoPwdSecretKey = "automation-agent-password"
+)
+
 // AutomationConfig maintains the raw map in the Deployment field
 // and constructs structs to make use of go's type safety
 // Dev notes: actually, this object is just a wrapper for the `Deployment` object which is received from Ops Manager,
@@ -426,19 +436,72 @@ func (ac *AutomationConfig) EnsureKeyFileContents() error {
 	return nil
 }
 
+// AuthSecretName for a given mdbName (`mdbName`) returns the name of
+// the secret associated with it.
+func AuthSecretName(mdbName string) string {
+	return fmt.Sprintf("%s-agent-auth-secret", mdbName)
+}
+
 // EnsurePassword makes sure that there is an Automation Agent password
 // that the agents will use to communicate with the deployments. The password
 // is returned, so it can be provided to the other agents
-func (ac *AutomationConfig) EnsurePassword() (string, error) {
-	if ac.Auth.AutoPwd == "" || ac.Auth.AutoPwd == util.InvalidAutomationAgentPassword {
-		automationAgentBackupPassword, err := generate.KeyFileContents()
+// EnsurePassword makes sure that there is an Automation Agent password
+// that the agents will use to communicate with the deployments. The password
+// is returned, so it can be provided to the other agents.
+func (ac *AutomationConfig) EnsurePassword(k8sClient secret.GetUpdateCreator, ctx context.Context, mdbNamespacedName *types.NamespacedName) (string, error) {
+	secretName := AuthSecretName(mdbNamespacedName.Name)
+	secretNamespacedName := client.ObjectKey{Name: secretName, Namespace: mdbNamespacedName.Namespace}
+	var password string
+
+	data, err := secret.ReadStringData(ctx, k8sClient, secretNamespacedName)
+	if err == nil {
+		if val, ok := data[autoPwdSecretKey]; ok && len(val) > 0 {
+			password = val
+		}
+	} else if secret.SecretNotExist(err) {
+		if ac.Auth.AutoPwd != "" && ac.Auth.AutoPwd != util.InvalidAutomationAgentPassword {
+			password = ac.Auth.AutoPwd
+		}
+
+		err := EnsureEmptySecret(ctx, k8sClient, secretNamespacedName)
 		if err != nil {
 			return "", err
 		}
-		ac.Auth.AutoPwd = automationAgentBackupPassword
-		return automationAgentBackupPassword, nil
 	}
-	return ac.Auth.AutoPwd, nil
+
+	if password == "" {
+		generatedPassword, genErr := generate.KeyFileContents()
+		if genErr != nil {
+			return "", genErr
+		}
+		password = generatedPassword
+	}
+
+	ac.Auth.AutoPwd = password
+	err = secret.UpdateField(ctx, k8sClient, secretNamespacedName, autoPwdSecretKey, password)
+	if err != nil {
+		return "", fmt.Errorf("failed to update password field in shared secret %s/%s: %w", secretNamespacedName.Namespace, secretNamespacedName.Name, err)
+	}
+
+	return password, nil
+}
+
+func EnsureEmptySecret(ctx context.Context, k8sClient secret.GetUpdateCreator, secretNamespacedName types.NamespacedName) error {
+	dataFields := map[string]string{
+		autoPwdSecretKey: "",
+	}
+
+	emptySecret := secret.Builder().
+		SetName(secretNamespacedName.Name).
+		SetNamespace(secretNamespacedName.Namespace).
+		SetStringMapToData(dataFields).
+		Build()
+
+	if err := secret.CreateOrUpdateIfNeeded(ctx, k8sClient, emptySecret); err != nil {
+		return fmt.Errorf("failed to create or update empty secret %s/%s: %w", secretNamespacedName.Namespace, secretNamespacedName.Name, err)
+	}
+
+	return nil
 }
 
 func (ac *AutomationConfig) CanEnableX509ProjectAuthentication() (bool, string) {

controllers/operator/appdbreplicaset_controller.go

@@ -1667,7 +1667,7 @@ func (r *ReconcileAppDbReplicaSet) tryConfigureMonitoringInOpsManager(ctx contex
 		AutoPEMKeyFilePath: agentCertPath,
 		CAFilePath:         util.CAFilePathInContainer,
 	}
-	err = authentication.Configure(conn, opts, false, log)
+	err = authentication.Configure(r.client, ctx, &types.NamespacedName{Namespace: opsManager.Namespace, Name: opsManager.Name}, conn, opts, false, log)
 	if err != nil {
 		log.Errorf("Could not set Automation Authentication options in Ops/Cloud Manager for the Application Database. "+
 			"Application Database is always configured with authentication enabled, but this will not be "+

controllers/operator/authentication/authentication.go

@@ -1,13 +1,17 @@
 package authentication
 
 import (
+	"context"
+
 	"go.uber.org/zap"
 	"golang.org/x/xerrors"
+	"k8s.io/apimachinery/pkg/types"
 
 	mdbv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdb"
 	"github.com/mongodb/mongodb-kubernetes/controllers/om"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/ldap"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/oidc"
+	kubernetesClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util"
 )
 
@@ -82,7 +86,7 @@ type UserOptions struct {
 
 // Configure will configure all the specified authentication Mechanisms. We need to ensure we wait for
 // the agents to reach ready state after each operation as prematurely updating the automation config can cause the agents to get stuck.
-func Configure(conn om.Connection, opts Options, isRecovering bool, log *zap.SugaredLogger) error {
+func Configure(client kubernetesClient.Client, ctx context.Context, mdbNamespacedName *types.NamespacedName, conn om.Connection, opts Options, isRecovering bool, log *zap.SugaredLogger) error {
 	log.Infow("ensuring correct deployment mechanisms", "ProcessNames", opts.ProcessNames, "Mechanisms", opts.Mechanisms)
 
 	// In case we're recovering, we can push all changes at once, because the mechanism is triggered after 20min by default.
@@ -113,7 +117,7 @@ func Configure(conn om.Connection, opts Options, isRecovering bool, log *zap.Sug
 
 	// once we have made sure that the deployment authentication mechanism array contains the desired auth mechanism
 	// we can then configure the agent authentication.
-	if err := enableAgentAuthentication(conn, opts, log); err != nil {
+	if err := enableAgentAuthentication(client, ctx, mdbNamespacedName, conn, opts, log); err != nil {
 		return xerrors.Errorf("error enabling agent authentication: %w", err)
 	}
 	if err := waitForReadyStateIfNeeded(); err != nil {
@@ -151,7 +155,7 @@ func Configure(conn om.Connection, opts Options, isRecovering bool, log *zap.Sug
 
 // Disable disables all authentication mechanisms, and waits for the agents to reach goal state. It is still required to provide
 // automation agent username, password and keyfile contents to ensure a valid Automation Config.
-func Disable(conn om.Connection, opts Options, deleteUsers bool, log *zap.SugaredLogger) error {
+func Disable(client kubernetesClient.Client, ctx context.Context, mdbNamespacedName *types.NamespacedName, conn om.Connection, opts Options, deleteUsers bool, log *zap.SugaredLogger) error {
 	ac, err := conn.ReadAutomationConfig()
 	if err != nil {
 		return xerrors.Errorf("error reading automation config: %w", err)
@@ -181,7 +185,7 @@ func Disable(conn om.Connection, opts Options, deleteUsers bool, log *zap.Sugare
 		if err := ac.EnsureKeyFileContents(); err != nil {
 			return xerrors.Errorf("error ensuring keyfile contents: %w", err)
 		}
-		if _, err := ac.EnsurePassword(); err != nil {
+		if _, err := ac.EnsurePassword(client, ctx, mdbNamespacedName); err != nil {
 			return xerrors.Errorf("error ensuring agent password: %w", err)
 		}
 
@@ -258,7 +262,7 @@ func removeUnsupportedAgentMechanisms(conn om.Connection, opts Options, log *zap
 
 // enableAgentAuthentication determines which agent authentication mechanism should be configured
 // and enables it in Ops Manager
-func enableAgentAuthentication(conn om.Connection, opts Options, log *zap.SugaredLogger) error {
+func enableAgentAuthentication(client kubernetesClient.Client, ctx context.Context, mdbNamespacedName *types.NamespacedName, conn om.Connection, opts Options, log *zap.SugaredLogger) error {
 	ac, err := conn.ReadAutomationConfig()
 	if err != nil {
 		return xerrors.Errorf("error reading automation config: %w", err)
@@ -267,7 +271,7 @@ func enableAgentAuthentication(conn om.Connection, opts Options, log *zap.Sugare
 	// we then configure the agent authentication for that type
 	mechanism := convertToMechanismOrPanic(opts.AgentMechanism, ac)
 
-	if err := ensureAgentAuthenticationIsConfigured(conn, opts, ac, mechanism, log); err != nil {
+	if err := ensureAgentAuthenticationIsConfigured(client, ctx, mdbNamespacedName, conn, opts, ac, mechanism, log); err != nil {
 		return xerrors.Errorf("error ensuring agent authentication is configured: %w", err)
 	}
 
@@ -365,14 +369,14 @@ func addOrRemoveAgentClientCertificate(conn om.Connection, opts Options, log *za
 }
 
 // ensureAgentAuthenticationIsConfigured will configure the agent authentication settings based on the desiredAgentAuthMechanism
-func ensureAgentAuthenticationIsConfigured(conn om.Connection, opts Options, ac *om.AutomationConfig, mechanism Mechanism, log *zap.SugaredLogger) error {
+func ensureAgentAuthenticationIsConfigured(client kubernetesClient.Client, ctx context.Context, mdbNamespacedName *types.NamespacedName, conn om.Connection, opts Options, ac *om.AutomationConfig, mechanism Mechanism, log *zap.SugaredLogger) error {
 	if mechanism.IsAgentAuthenticationConfigured(ac, opts) {
 		log.Infof("Agent authentication mechanism %s is already configured", mechanism.GetName())
 		return nil
 	}
 
 	log.Infof("Enabling %s agent authentication", mechanism.GetName())
-	return mechanism.EnableAgentAuthentication(conn, opts, log)
+	return mechanism.EnableAgentAuthentication(client, ctx, mdbNamespacedName, conn, opts, log)
 }
 
 // ensureDeploymentMechanisms configures the given AutomationConfig to allow deployments to

controllers/operator/authentication/authentication_mechanism.go

@@ -1,19 +1,22 @@
 package authentication
 
 import (
+	"context"
 	"slices"
 	"strings"
 
 	"go.uber.org/zap"
 	"golang.org/x/xerrors"
+	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/mongodb/mongodb-kubernetes/controllers/om"
+	kubernetesClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util"
 )
 
 // Mechanism is an interface that needs to be implemented for any Ops Manager authentication mechanism
 type Mechanism interface {
-	EnableAgentAuthentication(conn om.Connection, opts Options, log *zap.SugaredLogger) error
+	EnableAgentAuthentication(client kubernetesClient.Client, ctx context.Context, mdbNamespacedName *types.NamespacedName, conn om.Connection, opts Options, log *zap.SugaredLogger) error
 	DisableAgentAuthentication(conn om.Connection, log *zap.SugaredLogger) error
 	EnableDeploymentAuthentication(conn om.Connection, opts Options, log *zap.SugaredLogger) error
 	DisableDeploymentAuthentication(conn om.Connection, log *zap.SugaredLogger) error