[Feature] Abort resignation of leadership when dbserver restarted (#1291)

Author: informalict

CHANGELOG.md

@@ -6,6 +6,7 @@
 - (Feature) Run secured containers as a feature
 - (Feature) Expose core.PodSecurityContext Sysctl options
 - (Bugfix) Skip Collection check for missing Database
+- (Feature) Abort resignation of leadership when DB server is restared
 
 ## [1.2.31](https://github.com/arangodb/kube-arangodb/tree/1.2.31) (2023-07-14)
 - (Improvement) Block traffic on the services if there is more than 1 active leader in ActiveFailover mode

pkg/apis/deployment/v1/plan_locals.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,12 +20,18 @@
 
 package v1
 
+import "fmt"
+
 type PlanLocalKey string
 
 func (p PlanLocalKey) String() string {
 	return string(p)
 }
 
+func (p PlanLocalKey) Register(action Action, format string, args ...interface{}) Action {
+	return action.AddParam(p.String(), fmt.Sprintf(format, args...))
+}
+
 type PlanLocals map[PlanLocalKey]string
 
 func (p *PlanLocals) Remove(key PlanLocalKey) bool {

pkg/apis/deployment/v2alpha1/plan_locals.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,12 +20,18 @@
 
 package v2alpha1
 
+import "fmt"
+
 type PlanLocalKey string
 
 func (p PlanLocalKey) String() string {
 	return string(p)
 }
 
+func (p PlanLocalKey) Register(action Action, format string, args ...interface{}) Action {
+	return action.AddParam(p.String(), fmt.Sprintf(format, args...))
+}
+
 type PlanLocals map[PlanLocalKey]string
 
 func (p *PlanLocals) Remove(key PlanLocalKey) bool {

pkg/deployment/agency/definitions.go

@@ -36,6 +36,7 @@ const (
 	TargetKey  = "Target"
 
 	CurrentMaintenanceServers = "MaintenanceServers"
+	CurrentServersKnown       = "ServersKnown"
 
 	TargetHotBackupKey = "HotBackup"
 
@@ -64,10 +65,6 @@ func GetAgencyKey(parts ...string) string {
 	return fmt.Sprintf("/%s", strings.Join(parts, "/"))
 }
 
-func GetAgencyReadKey(elements ...string) []string {
-	return elements
-}
-
 func GetAgencyReadRequest(elements ...[]string) ReadRequest {
 	return elements
 }
@@ -78,6 +75,7 @@ func GetAgencyReadRequestFields() ReadRequest {
 		GetAgencyKey(ArangoKey, PlanKey, PlanCollectionsKey),
 		GetAgencyKey(ArangoKey, PlanKey, PlanDatabasesKey),
 		GetAgencyKey(ArangoKey, CurrentKey, PlanCollectionsKey),
+		GetAgencyKey(ArangoKey, CurrentKey, CurrentServersKnown),
 		GetAgencyKey(ArangoKey, CurrentKey, CurrentMaintenanceServers),
 		GetAgencyKey(ArangoKey, TargetKey, TargetHotBackupKey),
 		GetAgencyKey(ArangoKey, TargetKey, TargetJobToDoKey),

pkg/deployment/agency/state/state.go

@@ -20,6 +20,8 @@
 
 package state
 
+import "github.com/arangodb/go-driver"
+
 type Root struct {
 	Arango   State `json:"arango"`
 	ArangoDB DB    `json:"arangodb,omitempty"`
@@ -40,9 +42,18 @@ type State struct {
 	Target      Target      `json:"Target"`
 }
 
+// ServerKnown stores information about single ArangoDB server.
+type ServerKnown struct {
+	// RebootID is an incremental value which describes how many times server was restarted.
+	RebootID int `json:"rebootId"`
+}
+
 type Current struct {
 	MaintenanceServers CurrentMaintenanceServers `json:"MaintenanceServers,omitempty"`
 	Collections        CurrentCollections        `json:"Collections"`
+
+	// ServersKnown stores information about ArangoDB servers.
+	ServersKnown map[driver.ServerID]ServerKnown `json:"ServersKnown,omitempty"`
 }
 
 type Plan struct {
@@ -371,3 +382,13 @@ func (s State) GetCollectionDatabaseByID(id string) (string, bool) {
 
 	return "", false
 }
+
+// GetRebootID returns reboot ID for a given server ID.
+// returns false when a server ID does not exist in cache.
+func (s State) GetRebootID(id driver.ServerID) (int, bool) {
+	if v, ok := s.Current.ServersKnown[id]; ok {
+		return v.RebootID, true
+	}
+
+	return 0, false
+}

pkg/deployment/agency/state/state_test.go

@@ -373,3 +373,20 @@ func Test_MissingDatabaseCase(t *testing.T) {
 
 	require.Len(t, GetDBServerBlockingRestartShards(s, "PRMR-1e4bxazq"), 0)
 }
+
+func Test_GetRebootID(t *testing.T) {
+	var s DumpState
+	require.NoError(t, json.Unmarshal(agencyDump39, &s))
+
+	t.Run("Existing", func(t *testing.T) {
+		id, ok := s.Agency.Arango.GetRebootID("PRMR-n92yizyp")
+		require.True(t, ok)
+		require.Equal(t, 1, id)
+	})
+
+	t.Run("Missing", func(t *testing.T) {
+		id, ok := s.Agency.Arango.GetRebootID("PRMR-n92yiz")
+		require.False(t, ok)
+		require.Equal(t, 0, id)
+	})
+}

pkg/deployment/reconcile/action_resign_leadership.go

@@ -22,6 +22,7 @@ package reconcile
 
 import (
 	"context"
+	"strconv"
 
 	"github.com/arangodb/go-driver"
 
@@ -31,6 +32,10 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/util/globals"
 )
 
+const (
+	actionResignLeadershipRebootID api.PlanLocalKey = "rebootID"
+)
+
 // newResignLeadershipAction creates a new Action that implements the given
 // planned ResignLeadership action.
 func newResignLeadershipAction(action api.Action, actionCtx ActionContext) Action {
@@ -63,14 +68,14 @@ func (a *actionResignLeadership) Start(ctx context.Context) (bool, error) {
 	client, err := a.actionCtx.GetMembersState().State().GetDatabaseClient()
 	if err != nil {
 		a.log.Err(err).Error("Unable to get client")
-		return true, errors.WithStack(err)
+		return false, errors.WithStack(err)
 	}
 
 	switch group {
 	case api.ServerGroupDBServers:
 		if agencyState, agencyOK := a.actionCtx.GetAgencyCache(); !agencyOK {
-			a.log.Err(err).Warn("Maintenance is enabled, skipping action")
-			return true, errors.WithStack(err)
+			a.log.Warn("AgencyCache is not ready")
+			return false, nil
 		} else if agencyState.Supervision.Maintenance.Exists() {
 			// We are done, action cannot be handled on maintenance mode
 			a.log.Warn("Maintenance is enabled, skipping action")
@@ -82,7 +87,7 @@ func (a *actionResignLeadership) Start(ctx context.Context) (bool, error) {
 		cluster, err := client.Cluster(ctxChild)
 		if err != nil {
 			a.log.Err(err).Error("Unable to get cluster client")
-			return true, errors.WithStack(err)
+			return false, errors.WithStack(err)
 		}
 
 		var jobID string
@@ -92,13 +97,13 @@ func (a *actionResignLeadership) Start(ctx context.Context) (bool, error) {
 		a.log.Debug("Temporary shutdown, resign leadership")
 		if err := cluster.ResignServer(jobCtx, m.ID); err != nil {
 			a.log.Err(err).Debug("Failed to resign server")
-			return true, errors.WithStack(err)
+			return false, errors.WithStack(err)
 		}
 
 		m.CleanoutJobID = jobID
 
 		if err := a.actionCtx.UpdateMember(ctx, m); err != nil {
-			return true, errors.WithStack(err)
+			return false, errors.WithStack(err)
 		}
 
 		return false, nil
@@ -127,6 +132,8 @@ func (a *actionResignLeadership) CheckProgress(ctx context.Context) (bool, bool,
 			return false, false, errors.WithStack(err)
 		}
 		return true, false, nil
+	} else if a.isServerRebooted(agencyState, driver.ServerID(m.ID)) {
+		return true, false, nil
 	}
 
 	_, jobStatus := agencyState.Target.GetJob(state.JobID(m.CleanoutJobID))
@@ -150,3 +157,31 @@ func (a *actionResignLeadership) CheckProgress(ctx context.Context) (bool, bool,
 	}
 	return false, false, nil
 }
+
+// isServerRebooted returns true when a given server ID was rebooted during resignation of leadership.
+func (a *actionResignLeadership) isServerRebooted(agencyState state.State, serverID driver.ServerID) bool {
+	rebootID, ok := agencyState.GetRebootID(serverID)
+	if !ok {
+		return false
+	}
+
+	v, ok := a.actionCtx.Get(a.action, actionResignLeadershipRebootID)
+	if !ok {
+		a.log.Warn("missing reboot ID in action's locals", v)
+		return false
+	}
+
+	r, err := strconv.Atoi(v)
+	if err != nil {
+		a.log.Err(err).Warn("reboot ID '%s' supposed to be a number", v)
+		return false
+	}
+
+	if rebootID <= r {
+		// Server has not been restarted.
+		return false
+	}
+
+	a.log.Warn("resign leadership aborted because rebootID has changed from %d to %d", r, rebootID)
+	return true
+}

pkg/deployment/reconcile/helper_wrap.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -59,12 +59,17 @@ func withMemberMaintenance(group api.ServerGroup, member api.MemberStatus, reaso
 		actions.NewAction(api.ActionTypeDisableMemberMaintenance, group, member, reason))
 }
 
-func withResignLeadership(group api.ServerGroup, member api.MemberStatus, reason string, plan api.Plan) api.Plan {
+func withResignLeadership(group api.ServerGroup, member api.MemberStatus, reason string, plan api.Plan, rebootID *int) api.Plan {
 	if member.Image == nil {
 		return plan
 	}
 
-	return api.AsPlan(plan).Before(actions.NewAction(api.ActionTypeResignLeadership, group, member, reason))
+	action := actions.NewAction(api.ActionTypeResignLeadership, group, member, reason)
+	if rebootID != nil {
+		action = actionResignLeadershipRebootID.Register(action, "%d", *rebootID)
+	}
+
+	return api.AsPlan(plan).Before(action)
 }
 
 func cleanOutMember(group api.ServerGroup, m api.MemberStatus) api.Plan {