Create event on plan timeout

Author: ewoutp

pkg/deployment/reconcile/context.go

@@ -54,6 +54,9 @@ type Context interface {
 	GetAgencyClients(ctx context.Context, predicate func(id string) bool) ([]driver.Connection, error)
 	// GetSyncServerClient returns a cached client for a specific arangosync server.
 	GetSyncServerClient(ctx context.Context, group api.ServerGroup, id string) (client.API, error)
+	// CreateEvent creates a given event.
+	// On error, the error is logged.
+	CreateEvent(evt *v1.Event)
 	// CreateMember adds a new member to the given group.
 	// If ID is non-empty, it will be used, otherwise a new ID is created.
 	CreateMember(group api.ServerGroup, id string) error

pkg/deployment/reconcile/plan_executor.go

@@ -27,10 +27,11 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/rs/zerolog"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
-	"github.com/rs/zerolog"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 )
 
 // ExecutePlan tries to execute the plan as far as possible.
@@ -112,8 +113,9 @@ func (d *Reconciler) ExecutePlan(ctx context.Context) (bool, error) {
 				if time.Now().After(deadline) {
 					// Timeout has expired
 					log.Warn().Msg("Action not finished in time. Removing the entire plan")
+					d.context.CreateEvent(k8sutil.NewPlanTimeoutEvent(d.context.GetAPIObject(), string(planAction.Type), planAction.MemberID, planAction.Group.AsRole()))
+					// Replace plan with empty one and save it.
 					status.Plan = api.Plan{}
-					// Save plan update
 					if err := d.context.UpdateStatus(status); err != nil {
 						log.Debug().Err(err).Msg("Failed to update CR status")
 						return false, maskAny(err)

pkg/deployment/resources/context.go

@@ -62,7 +62,7 @@ type Context interface {
 	GetLifecycleImage() string
 	// GetNamespace returns the namespace that contains the deployment
 	GetNamespace() string
-	// createEvent creates a given event.
+	// CreateEvent creates a given event.
 	// On error, the error is logged.
 	CreateEvent(evt *v1.Event)
 	// GetOwnedPods returns a list of all pods owned by the deployment.

pkg/util/k8sutil/events.go

@@ -145,6 +145,16 @@ func NewAccessPackageDeletedEvent(apiObject APIObject, apSecretName string) *v1.
 	return event
 }
 
+// NewPlanTimeoutEvent creates an event indicating that an item on a reconciliation plan did not
+// finish before its deadline.
+func NewPlanTimeoutEvent(apiObject APIObject, itemType, memberID, role string) *v1.Event {
+	event := newDeploymentEvent(apiObject)
+	event.Type = v1.EventTypeNormal
+	event.Reason = "Reconciliation Plan Timeout"
+	event.Message = fmt.Sprintf("An plan item of type %s or member %s with role %s did not finish in time", itemType, memberID, role)
+	return event
+}
+
 // NewErrorEvent creates an even of type error.
 func NewErrorEvent(reason string, err error, apiObject APIObject) *v1.Event {
 	event := newDeploymentEvent(apiObject)