[Feature] BackupInProgress & Maintenace Conditions (#956)

Author: ajanikow

CHANGELOG.md

@@ -7,6 +7,7 @@
 - (Bugfix) ArangoSync port fix
 - (Bugfix) Fix GetClient lock system
 - (Feature) Backup InProgress Agency key discovery
+- (Feature) Backup & Maintenance Conditions
 
 ## [1.2.9](https://github.com/arangodb/kube-arangodb/tree/1.2.9) (2022-03-30)
 - (Feature) Improve Kubernetes clientsets management

pkg/apis/deployment/v1/conditions.go

@@ -96,6 +96,12 @@ const (
 
 	// ConditionTypeLicenseSet indicates that license V2 is set on cluster.
 	ConditionTypeLicenseSet ConditionType = "LicenseSet"
+
+	// ConditionTypeBackupInProgress indicates that there is Backup in progress on cluster
+	ConditionTypeBackupInProgress ConditionType = "BackupInProgress"
+
+	// ConditionTypeMaintenance indicates that maintenance is enabled on cluster
+	ConditionTypeMaintenance ConditionType = "Maintenance"
 )
 
 // Condition represents one current condition of a deployment or deployment member.

pkg/apis/deployment/v2alpha1/conditions.go

@@ -96,6 +96,12 @@ const (
 
 	// ConditionTypeLicenseSet indicates that license V2 is set on cluster.
 	ConditionTypeLicenseSet ConditionType = "LicenseSet"
+
+	// ConditionTypeBackupInProgress indicates that there is Backup in progress on cluster
+	ConditionTypeBackupInProgress ConditionType = "BackupInProgress"
+
+	// ConditionTypeMaintenance indicates that maintenance is enabled on cluster
+	ConditionTypeMaintenance ConditionType = "Maintenance"
 )
 
 // Condition represents one current condition of a deployment or deployment member.

pkg/deployment/agency/state.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/arangodb/go-driver"
 	"github.com/arangodb/go-driver/agency"
+	"github.com/arangodb/kube-arangodb/pkg/util"
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 )
 
@@ -113,18 +114,31 @@ type StateSupervision struct {
 	Maintenance StateExists `json:"Maintenance,omitempty"`
 }
 
-type StateExists bool
+type StateExists []byte
 
-func (d *StateExists) Exists() bool {
+func (d StateExists) Hash() string {
 	if d == nil {
-		return false
+		return ""
 	}
 
-	return bool(*d)
+	return util.SHA256(d)
+}
+
+func (d StateExists) Exists() bool {
+	return d != nil
 }
 
 func (d *StateExists) UnmarshalJSON(bytes []byte) error {
-	*d = bytes != nil
+	if bytes == nil {
+		*d = nil
+		return nil
+	}
+
+	z := make([]byte, len(bytes))
+
+	copy(z, bytes)
+
+	*d = z
 	return nil
 }
 

pkg/deployment/agency/target.go

@@ -25,5 +25,5 @@ type StateTarget struct {
 }
 
 type StateTargetHotBackup struct {
-	Create *StateExists `json:"Create,omitempty"`
+	Create StateExists `json:"Create,omitempty"`
 }

pkg/deployment/reconcile/action_maintenance_condition.go

@@ -60,7 +60,7 @@ func (a *actionSetMaintenanceCondition) Start(ctx context.Context) (bool, error)
 	} else {
 
 		if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool {
-			if agencyState.Supervision.Maintenance {
+			if agencyState.Supervision.Maintenance.Exists() {
 				return s.Conditions.Update(api.ConditionTypeMaintenanceMode, true, "Maintenance", "Maintenance enabled")
 			} else {
 				return s.Conditions.Remove(api.ConditionTypeMaintenanceMode)

pkg/deployment/reconcile/action_resign_leadership.go

@@ -80,7 +80,7 @@ func (a *actionResignLeadership) Start(ctx context.Context) (bool, error) {
 		if agencyState, agencyOK := a.actionCtx.GetAgencyCache(); !agencyOK {
 			log.Warn().Err(err).Msgf("Maintenance is enabled, skipping action")
 			return true, errors.WithStack(err)
-		} else if agencyState.Supervision.Maintenance {
+		} else if agencyState.Supervision.Maintenance.Exists() {
 			// We are done, action cannot be handled on maintenance mode
 			log.Warn().Msgf("Maintenance is enabled, skipping action")
 			return true, nil
@@ -129,7 +129,7 @@ func (a *actionResignLeadership) CheckProgress(ctx context.Context) (bool, bool,
 	if agencyState, agencyOK := a.actionCtx.GetAgencyCache(); !agencyOK {
 		log.Error().Msgf("Unable to get maintenance mode")
 		return false, false, nil
-	} else if agencyState.Supervision.Maintenance {
+	} else if agencyState.Supervision.Maintenance.Exists() {
 		log.Warn().Msgf("Maintenance is enabled, skipping action")
 		// We are done, action cannot be handled on maintenance mode
 		m.CleanoutJobID = ""

pkg/deployment/reconcile/plan_builder_high.go

@@ -58,7 +58,9 @@ func createHighPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.A
 		ApplyIfEmptyWithBackOff(LicenseCheck, 30*time.Second, updateClusterLicense).
 		ApplyIfEmpty(createTopologyMemberConditionPlan).
 		ApplyIfEmpty(createRebalancerCheckPlan).
-		ApplyWithBackOff(BackOffCheck, time.Minute, emptyPlanBuilder))
+		ApplyWithBackOff(BackOffCheck, time.Minute, emptyPlanBuilder)).
+		Apply(createBackupInProgressConditionPlan). // Discover backups always
+		Apply(createMaintenanceConditionPlan)       // Discover maintenance always
 
 	return r.Plan(), r.BackOff(), true
 }

pkg/deployment/reconcile/plan_builder_maintenance.go

@@ -0,0 +1,128 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package reconcile
+
+import (
+	"context"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector"
+	"github.com/rs/zerolog"
+)
+
+func createBackupInProgressConditionPlan(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan {
+
+	if spec.Mode.Get() != api.DeploymentModeCluster {
+		return nil
+	}
+
+	cache, ok := context.GetAgencyCache()
+	if !ok {
+		return nil
+	}
+
+	currentCondition, currentConditionExists := status.Conditions.Get(api.ConditionTypeBackupInProgress)
+
+	backupInProgress := cache.Target.HotBackup.Create
+
+	if currentConditionExists {
+		// Condition exists
+		if !backupInProgress.Exists() {
+			// Condition needs to be removed
+			return api.Plan{
+				removeConditionActionV2("Backup not in progress", api.ConditionTypeBackupInProgress),
+			}
+		}
+
+		// Backup is in progress
+
+		hash := backupInProgress.Hash()
+
+		if !currentCondition.IsTrue() || currentCondition.Hash != hash {
+			return api.Plan{
+				updateConditionActionV2("Backup in progress", api.ConditionTypeBackupInProgress, true, "Backup In Progress", "", hash),
+			}
+		}
+
+		return nil
+	} else {
+		if backupInProgress.Exists() {
+			return api.Plan{
+				updateConditionActionV2("Backup in progress", api.ConditionTypeBackupInProgress, true, "Backup In Progress", "", backupInProgress.Hash()),
+			}
+		}
+
+		return nil
+	}
+}
+
+func createMaintenanceConditionPlan(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan {
+
+	if spec.Mode.Get() != api.DeploymentModeCluster {
+		return nil
+	}
+
+	cache, ok := context.GetAgencyCache()
+	if !ok {
+		return nil
+	}
+
+	currentCondition, currentConditionExists := status.Conditions.Get(api.ConditionTypeMaintenance)
+
+	backupInProgress := cache.Target.HotBackup.Create
+
+	if currentConditionExists {
+		// Condition exists
+		if !backupInProgress.Exists() {
+			// Condition needs to be removed
+			return api.Plan{
+				removeConditionActionV2("Backup not in progress", api.ConditionTypeMaintenance),
+			}
+		}
+
+		// Backup is in progress
+
+		hash := backupInProgress.Hash()
+
+		if !currentCondition.IsTrue() || currentCondition.Hash != hash {
+			return api.Plan{
+				updateConditionActionV2("Backup in progress", api.ConditionTypeMaintenance, true, "Backup In Progress", "", hash),
+			}
+		}
+
+		return nil
+	} else {
+		if backupInProgress.Exists() {
+			return api.Plan{
+				updateConditionActionV2("Backup in progress", api.ConditionTypeMaintenance, true, "Backup In Progress", "", backupInProgress.Hash()),
+			}
+		}
+
+		return nil
+	}
+}

pkg/deployment/reconcile/plan_builder_utils.go

@@ -77,6 +77,7 @@ func removeConditionActionV2(actionReason string, conditionType api.ConditionTyp
 		AddParam(setConditionActionV2KeyType, setConditionActionV2KeyTypeRemove)
 }
 
+//nolint:unparam
 func updateConditionActionV2(actionReason string, conditionType api.ConditionType, status bool, reason, message, hash string) api.Action {
 	statusBool := core.ConditionTrue
 	if !status {