Delete all coordinator pods if all coordinators are failed. If a pod is in strange phase but there is no plan, fail it.

Author: lamai93

pkg/apis/deployment/v1alpha/member_status_list.go

@@ -25,8 +25,10 @@ package v1alpha
 import (
 	"math/rand"
 	"sort"
+	"time"
 
 	"github.com/pkg/errors"
+	v1 "k8s.io/api/core/v1"
 )
 
 // MemberStatusList is a list of MemberStatus entries
@@ -178,3 +180,28 @@ func (l MemberStatusList) MembersReady() int {
 func (l MemberStatusList) AllMembersReady() bool {
 	return len(l) == l.MembersReady()
 }
+
+// AllConditionTrueSince returns true if all members satisfy the condition since the given period
+func (l MemberStatusList) AllConditionTrueSince(cond ConditionType, status v1.ConditionStatus, period time.Duration) bool {
+	trueCount := 0
+	for _, x := range l {
+		if c, ok := x.Conditions.Get(cond); ok {
+			if c.Status == status && c.LastTransitionTime.Time.Add(period).Before(time.Now()) {
+				trueCount++
+			}
+		}
+	}
+
+	return trueCount == len(l)
+}
+
+// AllFailed returns true if all members are failed
+func (l MemberStatusList) AllFailed() bool {
+	failedCount := 0
+	for _, x := range l {
+		if x.Phase.IsFailed() {
+			failedCount++
+		}
+	}
+	return failedCount == len(l)
+}

pkg/deployment/resilience/member_failure.go

@@ -49,11 +49,20 @@ func (r *Resilience) CheckMemberFailure() error {
 				Str("id", m.ID).
 				Str("role", group.AsRole()).
 				Logger()
-			// Check current state
-			if m.Phase != api.MemberPhaseCreated {
-				// Phase is not Created, so we're not looking further.
+
+			// Check if there are Members with Phase Upgrading or Rotation but no plan
+			switch m.Phase {
+			case api.MemberPhaseNone:
 				continue
+			case api.MemberPhaseUpgrading, api.MemberPhaseRotating, api.MemberPhaseCleanOut:
+				if len(status.Plan) == 0 {
+					log.Error().Msgf("No plan but member is in phase %s - marking as failed", m.Phase)
+					m.Phase = api.MemberPhaseFailed
+					status.Members.Update(m, group)
+					updateStatusNeeded = true
+				}
 			}
+
 			// Check if pod is ready
 			if m.Conditions.IsTrue(api.ConditionTypeReady) {
 				// Pod is now ready, so we're not looking further

pkg/deployment/resources/pod_inspector.go

@@ -188,13 +188,20 @@ func (r *Resources) InspectPods(ctx context.Context) (util.Interval, error) {
 					case api.MemberPhaseShuttingDown, api.MemberPhaseRotating, api.MemberPhaseUpgrading, api.MemberPhaseFailed:
 						// Shutdown was intended, so not need to do anything here.
 						// Just mark terminated
+						updateMemberNeeded := false
+						if m.Conditions.Update(api.ConditionTypeReady, false, "Pod Does Not Exist", "") {
+							updateMemberNeeded = true
+						}
 						wasTerminated := m.Conditions.IsTrue(api.ConditionTypeTerminated)
 						if m.Conditions.Update(api.ConditionTypeTerminated, true, "Pod Terminated", "") {
 							if !wasTerminated {
 								// Record termination time
 								now := metav1.Now()
 								m.RecentTerminations = append(m.RecentTerminations, now)
 							}
+							updateMemberNeeded = true
+						}
+						if updateMemberNeeded {
 							// Save it
 							if err := status.Members.Update(m, group); err != nil {
 								return maskAny(err)
@@ -237,6 +244,19 @@ func (r *Resources) InspectPods(ctx context.Context) (util.Interval, error) {
 	allMembersReady := status.Members.AllMembersReady(spec.GetMode(), spec.Sync.IsEnabled())
 	status.Conditions.Update(api.ConditionTypeReady, allMembersReady, "", "")
 
+	if status.Members.Coordinators.AllFailed() {
+		log.Error().Msg("All coordinators failed - reset")
+		for _, m := range status.Members.Coordinators {
+			if err := r.context.DeletePod(m.PodName); err != nil {
+				log.Error().Err(err).Msg("Failed to delete pod")
+			}
+			m.Phase = api.MemberPhaseNone
+			if err := status.Members.Update(m, api.ServerGroupCoordinators); err != nil {
+				log.Error().Err(err).Msg("Failed to update member")
+			}
+		}
+	}
+
 	// Update conditions
 	if len(podNamesWithScheduleTimeout) > 0 {
 		if status.Conditions.Update(api.ConditionTypePodSchedulingFailure, true,