CLOUDP-328217: Automation agent password secret

.evergreen-tasks.yml

@@ -584,6 +584,11 @@ tasks:
     tags: [ "patch-run" ]
     commands:
       - func: "e2e_test"
+
+  - name: e2e_sharded_cluster_scram_sha_256_switch_project
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
 
   - name: e2e_sharded_cluster_scram_sha_1_user_connectivity
     tags: [ "patch-run" ]

.evergreen.yml

@@ -667,6 +667,7 @@ task_groups:
       - e2e_replica_set_scram_x509_ic_manual_certs
       - e2e_sharded_cluster_scram_sha_1_upgrade
       - e2e_sharded_cluster_scram_sha_256_user_connectivity
+      - e2e_sharded_cluster_scram_sha_256_switch_project
       - e2e_sharded_cluster_scram_sha_1_user_connectivity
       - e2e_sharded_cluster_scram_x509_ic_manual_certs
       - e2e_sharded_cluster_external_access

controllers/om/automation_config.go

@@ -1,19 +1,28 @@
 package om
 
 import (
+	"context"
 	"encoding/json"
+	"fmt"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/spf13/cast"
 	"k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/ldap"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/oidc"
+	"github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/secret"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util/generate"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util/maputil"
 )
 
+// The constants for the authentication secret
+const agentAuthenticationSecretSuffix = "-agent-auth-secret"
+const autoPwdSecretKey = "automation-agent-password"
+
 // AutomationConfig maintains the raw map in the Deployment field
 // and constructs structs to make use of go's type safety
 // Dev notes: actually, this object is just a wrapper for the `Deployment` object which is received from Ops Manager,
@@ -415,7 +424,7 @@ type MongoDbVersionConfig struct {
 }
 
 // EnsureKeyFileContents makes sure a valid keyfile is generated and used for internal cluster authentication
-func (ac *AutomationConfig) EnsureKeyFileContents() error {
+func (ac *AutomationConfig) EnsureKeyFileContents(k8sClient secret.GetUpdateCreator, ctx context.Context, mdbNamespacedName *types.NamespacedName) error {
 	if ac.Auth.Key == "" || ac.Auth.Key == util.InvalidKeyFileContents {
 		keyfileContents, err := generate.KeyFileContents()
 		if err != nil {
@@ -429,16 +438,63 @@ func (ac *AutomationConfig) EnsureKeyFileContents() error {
 // EnsurePassword makes sure that there is an Automation Agent password
 // that the agents will use to communicate with the deployments. The password
 // is returned, so it can be provided to the other agents
-func (ac *AutomationConfig) EnsurePassword() (string, error) {
-	if ac.Auth.AutoPwd == "" || ac.Auth.AutoPwd == util.InvalidAutomationAgentPassword {
-		automationAgentBackupPassword, err := generate.KeyFileContents()
+// EnsurePassword makes sure that there is an Automation Agent password
+// that the agents will use to communicate with the deployments. The password
+// is returned, so it can be provided to the other agents.
+func (ac *AutomationConfig) EnsurePassword(k8sClient secret.GetUpdateCreator, ctx context.Context, mdbNamespacedName *types.NamespacedName) (string, error) {
+	secretNamespacedName := client.ObjectKey{Name: mdbNamespacedName.Name + agentAuthenticationSecretSuffix, Namespace: mdbNamespacedName.Namespace}
+
+	var password string
+
+	data, err := secret.ReadStringData(ctx, k8sClient, secretNamespacedName)
+	if err == nil {
+		if val, ok := data[autoPwdSecretKey]; ok && len(val) > 0 {
+			password = val
+		}
+	} else if secret.SecretNotExist(err) {
+		if ac.Auth.AutoPwd != "" && ac.Auth.AutoPwd != util.InvalidAutomationAgentPassword {
+			password = ac.Auth.AutoPwd
+		}
+
+		err := EnsureEmptySecret(ctx, k8sClient, secretNamespacedName)
 		if err != nil {
 			return "", err
 		}
-		ac.Auth.AutoPwd = automationAgentBackupPassword
-		return automationAgentBackupPassword, nil
 	}
-	return ac.Auth.AutoPwd, nil
+
+	if password == "" {
+		generatedPassword, genErr := generate.KeyFileContents()
+		if genErr != nil {
+			return "", genErr
+		}
+		password = generatedPassword
+	}
+
+	ac.Auth.AutoPwd = password
+	err = secret.UpdateField(ctx, k8sClient, secretNamespacedName, autoPwdSecretKey, password)
+	if err != nil {
+		return "", fmt.Errorf("failed to update password field in shared secret %s/%s: %w", secretNamespacedName.Namespace, secretNamespacedName.Name, err)
+	}
+
+	return password, nil
+}
+
+func EnsureEmptySecret(ctx context.Context, k8sClient secret.GetUpdateCreator, secretNamespacedName types.NamespacedName) error {
+	dataFields := map[string]string{
+		autoPwdSecretKey: "",
+	}
+
+	emptySecret := secret.Builder().
+		SetName(secretNamespacedName.Name).
+		SetNamespace(secretNamespacedName.Namespace).
+		SetStringMapToData(dataFields).
+		Build()
+
+	if err := secret.CreateOrUpdateIfNeeded(ctx, k8sClient, emptySecret); err != nil {
+		return fmt.Errorf("failed to create or update empty secret %s/%s: %w", secretNamespacedName.Namespace, secretNamespacedName.Name, err)
+	}
+
+	return nil
 }
 
 func (ac *AutomationConfig) CanEnableX509ProjectAuthentication() (bool, string) {

controllers/om/deployment.go

@@ -4,6 +4,7 @@ import (
 	"encoding/gob"
 	"encoding/json"
 	"fmt"
+	"hash/fnv"
 	"math"
 	"regexp"
 
@@ -468,6 +469,27 @@ func (d Deployment) GetProcessNames(kind interface{}, name string) []string {
 	}
 }
 
+// GenerateFNVSecretIdentifier creates a short, Kubernetes-compliant identifier with "secret-" prefix for the Deployment.
+func (d Deployment) GenerateFNVSecretIdentifier() (string, error) {
+	// Serialize the Deployment structure to JSON
+	serializedDeployment, err := json.Marshal(d.ToCanonicalForm())
+	if err != nil {
+		return "", fmt.Errorf("failed to serialize Deployment: %w", err)
+	}
+
+	// Use FNV-1a hash algorithm
+	hash := fnv.New32a() // Initialize FNV-1a for 32-bit hash
+	_, err = hash.Write(serializedDeployment)
+	if err != nil {
+		return "", fmt.Errorf("failed to compute hash: %w", err)
+	}
+
+	// Convert the hash into a hex string
+	identifier := fmt.Sprintf("secret-%x", hash.Sum32()) // Prepend prefix "secret-" to the hash
+
+	return identifier, nil
+}
+
 // ConfigureInternalClusterAuthentication configures all processes in processNames to have the corresponding
 // clusterAuthenticationMode enabled
 func (d Deployment) ConfigureInternalClusterAuthentication(processNames []string, clusterAuthenticationMode string, internalClusterPath string) {

controllers/operator/appdbreplicaset_controller.go

@@ -1667,7 +1667,7 @@ func (r *ReconcileAppDbReplicaSet) tryConfigureMonitoringInOpsManager(ctx contex
 		AutoPEMKeyFilePath: agentCertPath,
 		CAFilePath:         util.CAFilePathInContainer,
 	}
-	err = authentication.Configure(conn, opts, false, log)
+	err = authentication.Configure(r.client, ctx, &types.NamespacedName{Namespace: opsManager.Namespace, Name: opsManager.Name}, conn, opts, false, log)
 	if err != nil {
 		log.Errorf("Could not set Automation Authentication options in Ops/Cloud Manager for the Application Database. "+
 			"Application Database is always configured with authentication enabled, but this will not be "+

controllers/operator/authentication/authentication.go

@@ -1,13 +1,17 @@
 package authentication
 
 import (
+	"context"
+
 	"go.uber.org/zap"
 	"golang.org/x/xerrors"
+	"k8s.io/apimachinery/pkg/types"
 
 	mdbv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdb"
 	"github.com/mongodb/mongodb-kubernetes/controllers/om"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/ldap"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/oidc"
+	kubernetesClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util"
 )
 
@@ -82,7 +86,7 @@ type UserOptions struct {
 
 // Configure will configure all the specified authentication Mechanisms. We need to ensure we wait for
 // the agents to reach ready state after each operation as prematurely updating the automation config can cause the agents to get stuck.
-func Configure(conn om.Connection, opts Options, isRecovering bool, log *zap.SugaredLogger) error {
+func Configure(client kubernetesClient.Client, ctx context.Context, namespacedName *types.NamespacedName, conn om.Connection, opts Options, isRecovering bool, log *zap.SugaredLogger) error {
 	log.Infow("ensuring correct deployment mechanisms", "ProcessNames", opts.ProcessNames, "Mechanisms", opts.Mechanisms)
 
 	// In case we're recovering, we can push all changes at once, because the mechanism is triggered after 20min by default.
@@ -113,7 +117,7 @@ func Configure(conn om.Connection, opts Options, isRecovering bool, log *zap.Sug
 
 	// once we have made sure that the deployment authentication mechanism array contains the desired auth mechanism
 	// we can then configure the agent authentication.
-	if err := enableAgentAuthentication(conn, opts, log); err != nil {
+	if err := enableAgentAuthentication(client, ctx, namespacedName, conn, opts, log); err != nil {
 		return xerrors.Errorf("error enabling agent authentication: %w", err)
 	}
 	if err := waitForReadyStateIfNeeded(); err != nil {
@@ -151,7 +155,7 @@ func Configure(conn om.Connection, opts Options, isRecovering bool, log *zap.Sug
 
 // Disable disables all authentication mechanisms, and waits for the agents to reach goal state. It is still required to provide
 // automation agent username, password and keyfile contents to ensure a valid Automation Config.
-func Disable(conn om.Connection, opts Options, deleteUsers bool, log *zap.SugaredLogger) error {
+func Disable(client kubernetesClient.Client, ctx context.Context, namespacedName *types.NamespacedName, conn om.Connection, opts Options, deleteUsers bool, log *zap.SugaredLogger) error {
 	ac, err := conn.ReadAutomationConfig()
 	if err != nil {
 		return xerrors.Errorf("error reading automation config: %w", err)
@@ -178,10 +182,10 @@ func Disable(conn om.Connection, opts Options, deleteUsers bool, log *zap.Sugare
 	}
 
 	err = conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error {
-		if err := ac.EnsureKeyFileContents(); err != nil {
+		if err := ac.EnsureKeyFileContents(client, ctx, namespacedName); err != nil {
 			return xerrors.Errorf("error ensuring keyfile contents: %w", err)
 		}
-		if _, err := ac.EnsurePassword(); err != nil {
+		if _, err := ac.EnsurePassword(client, ctx, namespacedName); err != nil {
 			return xerrors.Errorf("error ensuring agent password: %w", err)
 		}
 
@@ -258,7 +262,7 @@ func removeUnsupportedAgentMechanisms(conn om.Connection, opts Options, log *zap
 
 // enableAgentAuthentication determines which agent authentication mechanism should be configured
 // and enables it in Ops Manager
-func enableAgentAuthentication(conn om.Connection, opts Options, log *zap.SugaredLogger) error {
+func enableAgentAuthentication(client kubernetesClient.Client, ctx context.Context, namespacedName *types.NamespacedName, conn om.Connection, opts Options, log *zap.SugaredLogger) error {
 	ac, err := conn.ReadAutomationConfig()
 	if err != nil {
 		return xerrors.Errorf("error reading automation config: %w", err)
@@ -267,7 +271,7 @@ func enableAgentAuthentication(conn om.Connection, opts Options, log *zap.Sugare
 	// we then configure the agent authentication for that type
 	mechanism := convertToMechanismOrPanic(opts.AgentMechanism, ac)
 
-	if err := ensureAgentAuthenticationIsConfigured(conn, opts, ac, mechanism, log); err != nil {
+	if err := ensureAgentAuthenticationIsConfigured(client, ctx, namespacedName, conn, opts, ac, mechanism, log); err != nil {
 		return xerrors.Errorf("error ensuring agent authentication is configured: %w", err)
 	}
 
@@ -365,14 +369,14 @@ func addOrRemoveAgentClientCertificate(conn om.Connection, opts Options, log *za
 }
 
 // ensureAgentAuthenticationIsConfigured will configure the agent authentication settings based on the desiredAgentAuthMechanism
-func ensureAgentAuthenticationIsConfigured(conn om.Connection, opts Options, ac *om.AutomationConfig, mechanism Mechanism, log *zap.SugaredLogger) error {
+func ensureAgentAuthenticationIsConfigured(client kubernetesClient.Client, ctx context.Context, namespacedName *types.NamespacedName, conn om.Connection, opts Options, ac *om.AutomationConfig, mechanism Mechanism, log *zap.SugaredLogger) error {
 	if mechanism.IsAgentAuthenticationConfigured(ac, opts) {
 		log.Infof("Agent authentication mechanism %s is already configured", mechanism.GetName())
 		return nil
 	}
 
 	log.Infof("Enabling %s agent authentication", mechanism.GetName())
-	return mechanism.EnableAgentAuthentication(conn, opts, log)
+	return mechanism.EnableAgentAuthentication(client, ctx, namespacedName, conn, opts, log)
 }
 
 // ensureDeploymentMechanisms configures the given AutomationConfig to allow deployments to

controllers/operator/authentication/authentication_mechanism.go

@@ -1,19 +1,22 @@
 package authentication
 
 import (
+	"context"
 	"slices"
 	"strings"
 
 	"go.uber.org/zap"
 	"golang.org/x/xerrors"
+	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/mongodb/mongodb-kubernetes/controllers/om"
+	kubernetesClient "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/client"
 	"github.com/mongodb/mongodb-kubernetes/pkg/util"
 )
 
 // Mechanism is an interface that needs to be implemented for any Ops Manager authentication mechanism
 type Mechanism interface {
-	EnableAgentAuthentication(conn om.Connection, opts Options, log *zap.SugaredLogger) error
+	EnableAgentAuthentication(client kubernetesClient.Client, ctx context.Context, namespacedName *types.NamespacedName, conn om.Connection, opts Options, log *zap.SugaredLogger) error
 	DisableAgentAuthentication(conn om.Connection, log *zap.SugaredLogger) error
 	EnableDeploymentAuthentication(conn om.Connection, opts Options, log *zap.SugaredLogger) error
 	DisableDeploymentAuthentication(conn om.Connection, log *zap.SugaredLogger) error