If server is terminated before drained, declare it as cleaned out and replace it with a new one.

Author: lamai93

pkg/deployment/resources/pod_finalizers.go

@@ -123,7 +123,7 @@ func (r *Resources) inspectFinalizerPodDrainDBServer(ctx context.Context, log ze
 	}
 
 	// If this DBServer is cleaned out, we need to remove the PVC.
-	if memberStatus.Conditions.IsTrue(api.ConditionTypeCleanedOut) {
+	if memberStatus.Conditions.IsTrue(api.ConditionTypeCleanedOut) || memberStatus.Phase == api.MemberPhaseDrain {
 		pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(r.context.GetNamespace())
 		if err := pvcs.Delete(memberStatus.PersistentVolumeClaimName, &metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) {
 			log.Warn().Err(err).Msg("Failed to delete PVC for member")

pkg/deployment/resources/pod_termination.go

@@ -189,6 +189,15 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol
 	// Not cleaned out yet, check member status
 	if memberStatus.Conditions.IsTrue(api.ConditionTypeTerminated) || (memberStatus.Phase == api.MemberPhaseCreated && !k8sutil.IsPodReady(p)) {
 		log.Warn().Msg("Member is already terminated before it could be cleaned out. Not good, but removing dbserver pod because we cannot do anything further")
+		// At this point we have to set CleanedOut to true,
+		// because we can no longer reason about the state in the agency and
+		// bringing back the dbserver again may result in an cleaned out server without us knowing
+		memberStatus.Conditions.Update(api.ConditionTypeCleanedOut, true, "Draining server failed", "")
+		memberStatus.Phase = api.MemberPhaseCreated
+		memberStatus.CleanoutJobID = ""
+		if err := updateMember(memberStatus); err != nil {
+			return maskAny(err)
+		}
 		return nil
 	}
 	if memberStatus.Phase == api.MemberPhaseCreated {