implement recreation of lost clusters still referenced by ClusterRequests

Author: Diaphteiros

internal/controllers/scheduler/controller.go

@@ -136,10 +136,35 @@ func (r *ClusterScheduler) handleCreateOrUpdate(ctx context.Context, req reconci
 
 	// check if request is already granted
 	if cr.Status.Cluster != nil {
-		log.Info("Request already contains a cluster reference, nothing to do", "clusterName", cr.Status.Cluster.Name, "clusterNamespace", cr.Status.Cluster.Namespace)
-		return rr
+		// verify that the referenced cluster still exists
+		existingCluster := &clustersv1alpha1.Cluster{}
+		existingCluster.Namespace = cr.Status.Cluster.Namespace
+		existingCluster.Name = cr.Status.Cluster.Name
+		if err := r.PlatformCluster.Client().Get(ctx, client.ObjectKeyFromObject(existingCluster), existingCluster); err == nil {
+			fin := cr.FinalizerForCluster()
+			if !controllerutil.ContainsFinalizer(existingCluster, fin) {
+				log.Info("Referenced cluster does not contain expected finalizer, adding it", "clusterName", existingCluster.Name, "clusterNamespace", existingCluster.Namespace, "finalizer", fin)
+				oldCluster := existingCluster.DeepCopy()
+				controllerutil.AddFinalizer(existingCluster, fin)
+				if err := r.PlatformCluster.Client().Patch(ctx, existingCluster, client.MergeFrom(oldCluster)); err != nil {
+					rr.ReconcileError = errutils.WithReason(fmt.Errorf("error patching finalizer '%s' on cluster '%s/%s': %w", fin, existingCluster.Namespace, existingCluster.Name, err), cconst.ReasonPlatformClusterInteractionProblem)
+					return rr
+				}
+			}
+			log.Info("Request already contains a reference to an existing cluster, nothing to do", "clusterName", cr.Status.Cluster.Name, "clusterNamespace", cr.Status.Cluster.Namespace)
+			return rr
+		} else if !apierrors.IsNotFound(err) {
+			rr.ReconcileError = errutils.WithReason(fmt.Errorf("error checking whether cluster '%s/%s' exists: %w", existingCluster.Namespace, existingCluster.Name, err), cconst.ReasonPlatformClusterInteractionProblem)
+			return rr
+		} else {
+			log.Info("Referenced cluster does not exist, resetting request status to recreate it", "clusterName", existingCluster.Name, "clusterNamespace", existingCluster.Namespace)
+			rr.Object.Status.Cluster = nil
+			rr.Result.RequeueAfter = 1
+			return rr
+		}
+	} else {
+		log.Debug("Request status does not contain a cluster reference, checking for existing clusters with referencing finalizers")
 	}
-	log.Debug("Request status does not contain a cluster reference, checking for existing clusters with referencing finalizers")
 
 	// fetch cluster definition
 	purpose := cr.Spec.Purpose

internal/controllers/scheduler/controller_test.go

@@ -538,7 +538,7 @@ var _ = Describe("Scheduler", func() {
 		Expect(cluster2.GenerateName).To(Equal(req2.Spec.Purpose + "-"))
 	})
 
-	It("should recreate a deleted cluster for an existing request", func() {
+	It("should restore a missing finalizer and recreate a deleted cluster for an existing request", func() {
 		clusterNamespace := "exclusive"
 		sc, env := defaultTestSetup("testdata", "test-01")
 		Expect(sc.Config.Scope).To(Equal(config.SCOPE_NAMESPACED))
@@ -561,6 +561,14 @@ var _ = Describe("Scheduler", func() {
 		Expect(cluster.Name).To(Equal(req.Status.Cluster.Name))
 		Expect(cluster.Namespace).To(Equal(clusterNamespace))
 
+		// remove finalizer from cluster and reconcile, which should add it again
+		cluster.Finalizers = []string{}
+		Expect(env.Client().Update(env.Ctx, &cluster)).To(Succeed())
+		Expect(cluster.Finalizers).To(BeEmpty())
+		env.ShouldReconcile(testutils.RequestFromObject(req))
+		Expect(env.Client().Get(env.Ctx, client.ObjectKeyFromObject(&cluster), &cluster)).To(Succeed())
+		Expect(cluster.Finalizers).To(ContainElements(req.FinalizerForCluster()), "Finalizer should be restored on cluster")
+
 		// remove all finalizers and delete the cluster
 		cluster.Finalizers = []string{}
 		Expect(env.Client().Update(env.Ctx, &cluster)).To(Succeed())
@@ -570,7 +578,14 @@ var _ = Describe("Scheduler", func() {
 			return apierrors.IsNotFound(err)
 		}, 3).Should(BeTrue(), "Cluster should be deleted")
 
-		// reconcile again, this should recreate the cluster
+		// reconciliation should remove the request status and set it to pending
+		rr := env.ShouldReconcile(testutils.RequestFromObject(req))
+		Expect(env.Client().Get(env.Ctx, client.ObjectKeyFromObject(req), req)).To(Succeed())
+		Expect(req.Status.Cluster).To(BeNil())
+		Expect(req.Status.Phase).To(Equal(clustersv1alpha1.REQUEST_PENDING))
+		Expect(rr.RequeueAfter).ToNot(BeZero(), "RequeueAfter should be set")
+
+		// reconcile again, this should re-create the cluster
 		env.ShouldReconcile(testutils.RequestFromObject(req))
 		Expect(env.Client().Get(env.Ctx, client.ObjectKeyFromObject(req), req)).To(Succeed())
 		Expect(req.Status.Cluster).ToNot(BeNil())