[Feature] Do not remove Agency members (#535)

Author: ajanikow

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Change Log
 
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
+- Prevent Agency Members recreation
 - Added Customizable Volumes and VolumeMounts for ArangoDB server container
 - Added MemoryOverride flag for ArangoDB >= 3.6.3
 - Improved Rotation discovery process

pkg/deployment/reconcile/action_recreate_member.go

@@ -63,7 +63,7 @@ func (a *actionRecreateMember) Start(ctx context.Context) (bool, error) {
 	_, err := a.actionCtx.GetPvc(m.PersistentVolumeClaimName)
 	if err != nil {
 		if kubeErrors.IsNotFound(err) {
-			return false, fmt.Errorf("PVC is missing %s. DBServer wont be recreated without old PV", m.PersistentVolumeClaimName)
+			return false, fmt.Errorf("PVC is missing %s. Members won't be recreated without old PV", m.PersistentVolumeClaimName)
 		}
 
 		return false, maskAny(err)

pkg/deployment/reconcile/plan_builder.go

@@ -155,15 +155,20 @@ func createPlan(log zerolog.Logger, apiObject k8sutil.APIObject,
 				// Everything is fine, proceed
 			}
 
-			memberLog.Msg("Creating member replacement plan because member has failed")
-			newID := ""
-			if group == api.ServerGroupAgents {
-				newID = m.ID // Agents cannot (yet) be replaced with new IDs
+			switch group {
+			case api.ServerGroupAgents:
+				// For agents just recreate member do not rotate ID, do not remove PVC or service
+				memberLog.Msg("Restoring old member. For agency members recreation of PVC is not supported - to prevent DataLoss")
+				plan = append(plan,
+					api.NewAction(api.ActionTypeRecreateMember, group, m.ID))
+			default:
+				memberLog.Msg("Creating member replacement plan because member has failed")
+				plan = append(plan,
+					api.NewAction(api.ActionTypeRemoveMember, group, m.ID),
+					api.NewAction(api.ActionTypeAddMember, group, ""),
+				)
+
 			}
-			plan = append(plan,
-				api.NewAction(api.ActionTypeRemoveMember, group, m.ID),
-				api.NewAction(api.ActionTypeAddMember, group, newID),
-			)
 		}
 		return nil
 	})

pkg/deployment/reconcile/plan_builder_test.go

@@ -681,7 +681,7 @@ func TestCreatePlan(t *testing.T) {
 			ExpectedLog: "Creating rotation plan",
 		},
 		{
-			Name: "Member in failed state",
+			Name: "Agent in failed state",
 			context: &testContext{
 				ArangoDeployment: deploymentTemplate.DeepCopy(),
 			},
@@ -690,10 +690,46 @@ func TestCreatePlan(t *testing.T) {
 					Count: util.NewInt(2),
 				}
 				ad.Status.Members.Agents[0].Phase = api.MemberPhaseFailed
+				ad.Status.Members.Agents[0].ID = "id"
 			},
 			ExpectedPlan: []api.Action{
-				api.NewAction(api.ActionTypeRemoveMember, api.ServerGroupAgents, ""),
-				api.NewAction(api.ActionTypeAddMember, api.ServerGroupAgents, ""),
+				api.NewAction(api.ActionTypeRecreateMember, api.ServerGroupAgents, "id"),
+			},
+			ExpectedLog: "Restoring old member. For agency members recreation of PVC is not supported - to prevent DataLoss",
+		},
+		{
+			Name: "Coordinator in failed state",
+			context: &testContext{
+				ArangoDeployment: deploymentTemplate.DeepCopy(),
+			},
+			Helper: func(ad *api.ArangoDeployment) {
+				ad.Spec.Coordinators = api.ServerGroupSpec{
+					Count: util.NewInt(2),
+				}
+				ad.Status.Members.Coordinators[0].Phase = api.MemberPhaseFailed
+				ad.Status.Members.Coordinators[0].ID = "id"
+			},
+			ExpectedPlan: []api.Action{
+				api.NewAction(api.ActionTypeRemoveMember, api.ServerGroupCoordinators, "id"),
+				api.NewAction(api.ActionTypeAddMember, api.ServerGroupCoordinators, ""),
+			},
+			ExpectedLog: "Creating member replacement plan because member has failed",
+		},
+		{
+			Name: "DBServer in failed state",
+			context: &testContext{
+				ArangoDeployment: deploymentTemplate.DeepCopy(),
+			},
+			Helper: func(ad *api.ArangoDeployment) {
+				ad.Spec.DBServers = api.ServerGroupSpec{
+					Count: util.NewInt(2),
+				}
+				ad.Status.Members.DBServers[0].Phase = api.MemberPhaseFailed
+				ad.Status.Members.DBServers[0].ID = "id"
+			},
+			ExpectedPlan: []api.Action{
+				api.NewAction(api.ActionTypeRemoveMember, api.ServerGroupDBServers, "id"),
+				api.NewAction(api.ActionTypeAddMember, api.ServerGroupDBServers, ""),
 			},
 			ExpectedLog: "Creating member replacement plan because member has failed",
 		},