Update member status. If once decided to drain, continue draining.

Author: lamai93

pkg/deployment/resources/pod_termination.go

@@ -123,6 +123,12 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol
 		log.Debug().Msg("Pod is already failed, safe to remove dbserver pod")
 		return nil
 	}
+	// If pod is not member of cluster, do nothing
+	if !memberStatus.Conditions.IsTrue(api.ConditionTypeMemberOfCluster) {
+		log.Debug().Msg("Pod is not member of cluster")
+		return nil
+	}
+
 	// Inspect deployment deletion state
 	apiObject := r.context.GetAPIObject()
 	if apiObject.GetDeletionTimestamp() != nil {
@@ -154,6 +160,11 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol
 		dbserverDataWillBeGone = true
 	}
 
+	// Once decided to drain the member, never go back
+	if memberStatus.Phase == api.MemberPhaseCreated {
+		dbserverDataWillBeGone = true
+	}
+
 	// Is this a simple pod restart?
 	if !dbserverDataWillBeGone {
 		log.Debug().Msg("Pod is just being restarted, safe to remove dbserver pod")
@@ -230,11 +241,15 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol
 			return maskAny(err)
 		}
 		if jobStatus.IsFailed() {
-			log.Warn().Str("reason", jobStatus.Reason()).Msg("Cleanout Job failed. Aborting plan")
+			log.Warn().Str("reason", jobStatus.Reason()).Msg("Cleanout Job failed")
 			// Revert cleanout state
 			memberStatus.Phase = api.MemberPhaseCreated
 			memberStatus.CleanoutJobID = ""
-			return maskAny(fmt.Errorf("Clean out server job failed"))
+			if err := updateMember(memberStatus); err != nil {
+				return maskAny(err)
+			}
+			log.Error().Msg("Cleanout server job failed, continue anyway")
+			return nil
 		}
 	}