Wait cluster ready before creating user or database

Signed-off-by: drivebyer <yang.wu@daocloud.io>

Author: drivebyer

pkg/controller/mysqldatabase/db_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"time"
 
 	"github.com/go-test/deep"
 	logf "github.com/presslabs/controller-util/log"
@@ -69,6 +70,7 @@ var _ reconcile.Reconciler = &ReconcileMySQLDatabase{}
 
 // Reconcile reads that state of the cluster for a Wordpress object and makes changes based on the state read
 // and what is in the Wordpress.Spec
+// nolint: gocyclo
 func (r *ReconcileMySQLDatabase) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
 	// Fetch the MysqlDatabase instance
 	db := mysqldatabase.Wrap(&mysqlv1alpha1.MysqlDatabase{})
@@ -105,6 +107,23 @@ func (r *ReconcileMySQLDatabase) Reconcile(ctx context.Context, request reconcil
 		return reconcile.Result{}, r.Update(ctx, db.Unwrap())
 	}
 
+	cluster := mysqlcluster.New(&mysqlv1alpha1.MysqlCluster{})
+	if err = r.Get(ctx, db.GetClusterKey(), cluster.Unwrap()); err != nil {
+		return reconcile.Result{}, r.updateReadyCondition(ctx, oldDBStatus, db, err)
+	}
+
+	if !cluster.IsClusterReady() {
+
+		log.Error(r.updateReadyCondition(ctx, oldDBStatus, db, fmt.Errorf("cluster is not ready")),
+			"cluster is not ready when create database",
+			"cluster", cluster.GetNamespacedName())
+
+		// The MysqlCluster and MysqlDatabase are separate Custom Resources (CRs). We typically apply both CRs at the same time.
+		// If the MysqlCluster is not ready, the database creation will fail. The requeue time becomes exponential when an error is returned.
+		// Therefore, we return a nil error here and specify the RequeueAfter to avoid the exponential requeue time.
+		return reconcile.Result{RequeueAfter: 30 * time.Second}, nil
+	}
+
 	// reconcile database in mysql
 	err = r.createDatabase(ctx, db)
 	if err != nil {

pkg/controller/mysqluser/mysqluser_controller.go

@@ -97,6 +97,26 @@ func (r *ReconcileMySQLUser) Reconcile(ctx context.Context, request reconcile.Re
 		return reconcile.Result{}, r.removeUser(ctx, user)
 	}
 
+	cluster := mysqlcluster.New(&mysqlv1alpha1.MysqlCluster{})
+	if err = r.Get(ctx, user.GetClusterKey(), cluster.Unwrap()); err != nil {
+		return reconcile.Result{}, err
+	}
+
+	if !cluster.IsClusterReady() {
+		user.UpdateStatusCondition(
+			mysqlv1alpha1.MySQLUserReady, corev1.ConditionFalse,
+			mysqluser.ProvisionFailedReason, "cluster is not ready",
+		)
+		log.Error(r.updateStatusAndErr(ctx, user, oldStatus, fmt.Errorf("cluster is not ready")),
+			"cluster is not ready when create user",
+			"cluster", cluster.GetNamespacedName())
+
+		// The MysqlCluster and MysqlUser are separate Custom Resources (CRs). We typically apply both CRs at the same time.
+		// If the MysqlCluster is not ready, the user creation will fail. The requeue time becomes exponential when an error is returned.
+		// Therefore, we return a nil error here and specify the RequeueAfter to avoid the exponential requeue time.
+		return reconcile.Result{RequeueAfter: 30 * time.Second}, nil
+	}
+
 	// write the desired status into mysql cluster
 	ruErr := r.reconcileUserInCluster(ctx, user)
 	if err := r.updateStatusAndErr(ctx, user, oldStatus, ruErr); err != nil {

pkg/controller/mysqluser/mysqluser_controller_test.go

@@ -120,6 +120,7 @@ var _ = Describe("MySQL user controller", func() {
 			BeforeEach(func() {
 				// Create prerequisite resources
 				cluster = factories.NewMySQLCluster(
+					factories.WithClusterReadyCondition(),
 					factories.CreateMySQLClusterSecret(c, &corev1.Secret{}),
 					factories.CreateMySQLClusterInK8s(c),
 				)
@@ -201,6 +202,7 @@ var _ = Describe("MySQL user controller", func() {
 			BeforeEach(func() {
 				// Create prerequisite resources
 				cluster = factories.NewMySQLCluster(
+					factories.WithClusterReadyCondition(),
 					factories.CreateMySQLClusterSecret(c, &corev1.Secret{}),
 					factories.CreateMySQLClusterInK8s(c),
 				)
@@ -302,6 +304,7 @@ var _ = Describe("MySQL user controller", func() {
 
 				// Create prerequisite resources
 				cluster = factories.NewMySQLCluster(
+					factories.WithClusterReadyCondition(),
 					factories.CreateMySQLClusterSecret(c, &corev1.Secret{}),
 					factories.CreateMySQLClusterInK8s(c),
 				)
@@ -394,7 +397,9 @@ var _ = Describe("MySQL user controller", func() {
 			BeforeEach(func() {
 				// Create prerequisite resources
 				cluster = factories.NewMySQLCluster(
-					factories.CreateMySQLClusterSecret(c, &corev1.Secret{}), factories.CreateMySQLClusterInK8s(c),
+					factories.WithClusterReadyCondition(),
+					factories.CreateMySQLClusterSecret(c, &corev1.Secret{}),
+					factories.CreateMySQLClusterInK8s(c),
 				)
 
 				// The mysql user creation fails
@@ -444,6 +449,7 @@ var _ = Describe("MySQL user controller", func() {
 		BeforeEach(func() {
 			// Create prerequisite resources
 			cluster = factories.NewMySQLCluster(
+				factories.WithClusterReadyCondition(),
 				factories.CreateMySQLClusterSecret(c, &corev1.Secret{}),
 				factories.CreateMySQLClusterInK8s(c),
 			)

pkg/internal/mysqlcluster/mysqlcluster.go

@@ -277,6 +277,18 @@ func (c *MysqlCluster) GetSidecarImage() string {
 	return sidecarImage
 }
 
+// IsClusterReady checks if the cluster is ready or not.
+func (c *MysqlCluster) IsClusterReady() bool {
+	isReady := false
+	for _, cond := range c.Status.Conditions {
+		if cond.Type == api.ClusterConditionReady && cond.Status == core.ConditionTrue {
+			isReady = true
+			break
+		}
+	}
+	return isReady
+}
+
 // IsMysqlClusterKind for the given kind checks if CRD kind is for MysqlCluster CRD
 func IsMysqlClusterKind(kind string) bool {
 	switch kind {

pkg/testutil/factories/mysql_database.go

@@ -40,7 +40,7 @@ func WithMySQLCluster(ctx context.Context, cl client.Client, name string) MysqlD
 	cluster := NewMySQLCluster(func(cluster *mysqlv1alpha1.MysqlCluster) error {
 		cluster.Name = name
 		return nil
-	}, CreateMySQLClusterSecret(cl, &corev1.Secret{}), CreateMySQLClusterInK8s(cl))
+	}, CreateMySQLClusterSecret(cl, &corev1.Secret{}), WithClusterReadyCondition(), CreateMySQLClusterInK8s(cl))
 
 	return func(db *mysqldatabase.Database) error {
 		db.Spec.ClusterRef.Name = cluster.Name