If all members are dropped, create a new one.

Author: lamai93

pkg/deployment/deployment_inspector.go

@@ -121,6 +121,12 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval
 			d.CreateEvent(k8sutil.NewErrorEvent("Member failure detection failed", err, d.apiObject))
 		}
 
+		// Immediate actions
+		if err := d.reconciler.CheckDeployment(); err != nil {
+			hasError = true
+			d.CreateEvent(k8sutil.NewErrorEvent("Reconciler immediate actions failed", err, d.apiObject))
+		}
+
 		// Create scale/update plan
 		if err := d.reconciler.CreatePlan(); err != nil {
 			hasError = true

pkg/deployment/reconcile/reconciler.go

@@ -22,7 +22,10 @@
 
 package reconcile
 
-import "github.com/rs/zerolog"
+import (
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
+	"github.com/rs/zerolog"
+)
 
 // Reconciler is the service that takes care of bring the a deployment
 // in line with its (changed) specification.
@@ -38,3 +41,35 @@ func NewReconciler(log zerolog.Logger, context Context) *Reconciler {
 		context: context,
 	}
 }
+
+// CheckDeployment checks for obviously broken things and fixes them immediately
+func (r *Reconciler) CheckDeployment() error {
+	spec := r.context.GetSpec()
+	status, _ := r.context.GetStatus()
+
+	if spec.GetMode().HasCoordinators() {
+
+		// Check if there are coordinators
+		if len(status.Members.Coordinators) == 0 {
+			// No more coordinators! Take immediate action
+			r.log.Error().Msg("No Coordinator members! Create one member immediately")
+			_, err := r.context.CreateMember(api.ServerGroupCoordinators, "")
+			if err != nil {
+				return err
+			}
+		} else if status.Members.Coordinators.AllFailed() {
+			r.log.Error().Msg("All coordinators failed - reset")
+			for _, m := range status.Members.Coordinators {
+				if err := r.context.DeletePod(m.PodName); err != nil {
+					r.log.Error().Err(err).Msg("Failed to delete pod")
+				}
+				m.Phase = api.MemberPhaseNone
+				if err := status.Members.Update(m, api.ServerGroupCoordinators); err != nil {
+					r.log.Error().Err(err).Msg("Failed to update member")
+				}
+			}
+		}
+	}
+
+	return nil
+}