nodeclass add hash

Signed-off-by: helen <helenfrank@protonmail.com>

Author: helen-frank

pkg/apis/v1alpha1/ecsnodeclass.go

@@ -17,9 +17,11 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"fmt"
 	"log"
 	"strings"
 
+	"github.com/mitchellh/hashstructure/v2"
 	"github.com/samber/lo"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -206,6 +208,22 @@ type ECSNodeClass struct {
 	Status ECSNodeClassStatus `json:"status,omitempty"`
 }
 
+// We need to bump the ECSNodeClassHashVersion when we make an update to the ECSNodeClass CRD under these conditions:
+// 1. A field changes its default value for an existing field that is already hashed
+// 2. A field is added to the hash calculation with an already-set value
+// 3. A field is removed from the hash calculations
+const ECSNodeClassHashVersion = "v1"
+
+func (in *ECSNodeClass) Hash() string {
+	return fmt.Sprint(lo.Must(hashstructure.Hash([]interface{}{
+		in.Spec,
+	}, hashstructure.FormatV2, &hashstructure.HashOptions{
+		SlicesAsSets:    true,
+		IgnoreZeroValue: true,
+		ZeroNil:         true,
+	})))
+}
+
 // ImageFamily If an alias is specified, return alias, or be 'Custom' (enforced via validation).
 func (in *ECSNodeClass) ImageFamily() string {
 	if term, ok := lo.Find(in.Spec.ImageSelectorTerms, func(t ImageSelectorTerm) bool {

pkg/controllers/controllers.go

@@ -28,6 +28,7 @@ import (
 
 	nodeclaimgarbagecollection "github.com/cloudpilot-ai/karpenter-provider-alicloud/pkg/controllers/nodeclaim/garbagecollection"
 	nodeclaimtagging "github.com/cloudpilot-ai/karpenter-provider-alicloud/pkg/controllers/nodeclaim/tagging"
+	nodeclasshash "github.com/cloudpilot-ai/karpenter-provider-alicloud/pkg/controllers/nodeclass/hash"
 	controllerspricing "github.com/cloudpilot-ai/karpenter-provider-alicloud/pkg/controllers/providers/pricing"
 	"github.com/cloudpilot-ai/karpenter-provider-alicloud/pkg/providers/instance"
 	"github.com/cloudpilot-ai/karpenter-provider-alicloud/pkg/providers/pricing"
@@ -37,6 +38,7 @@ func NewControllers(ctx context.Context, mgr manager.Manager, clk clock.Clock, k
 	cloudProvider cloudprovider.CloudProvider, instanceProvider instance.Provider, pricingProvider pricing.Provider) []controller.Controller {
 
 	controllers := []controller.Controller{
+		nodeclasshash.NewController(kubeClient),
 		controllerspricing.NewController(pricingProvider),
 		nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider),
 		nodeclaimtagging.NewController(kubeClient, instanceProvider),

pkg/controllers/nodeclass/hash/controller.go

@@ -0,0 +1,117 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hash
+
+import (
+	"context"
+
+	"github.com/awslabs/operatorpkg/reasonable"
+	"github.com/samber/lo"
+	"go.uber.org/multierr"
+	"k8s.io/apimachinery/pkg/api/equality"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
+	"sigs.k8s.io/karpenter/pkg/operator/injection"
+
+	"github.com/cloudpilot-ai/karpenter-provider-alicloud/pkg/apis/v1alpha1"
+)
+
+type Controller struct {
+	kubeClient client.Client
+}
+
+func NewController(kubeClient client.Client) *Controller {
+	return &Controller{
+		kubeClient: kubeClient,
+	}
+}
+
+func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1alpha1.ECSNodeClass) (reconcile.Result, error) {
+	ctx = injection.WithControllerName(ctx, "nodeclass.hash")
+
+	stored := nodeClass.DeepCopy()
+
+	if nodeClass.Annotations[v1alpha1.AnnotationECSNodeClassHashVersion] != v1alpha1.ECSNodeClassHashVersion {
+		if err := c.updateNodeClaimHash(ctx, nodeClass); err != nil {
+			return reconcile.Result{}, err
+		}
+	}
+	nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{
+		v1alpha1.AnnotationECSNodeClassHash:        nodeClass.Hash(),
+		v1alpha1.AnnotationECSNodeClassHashVersion: v1alpha1.ECSNodeClassHashVersion,
+	})
+
+	if !equality.Semantic.DeepEqual(stored, nodeClass) {
+		if err := c.kubeClient.Patch(ctx, nodeClass, client.MergeFrom(stored)); err != nil {
+			return reconcile.Result{}, err
+		}
+	}
+
+	return reconcile.Result{}, nil
+}
+
+func (c *Controller) Register(_ context.Context, m manager.Manager) error {
+	return controllerruntime.NewControllerManagedBy(m).
+		Named("nodeclass.hash").
+		For(&v1alpha1.ECSNodeClass{}).
+		WithOptions(controller.Options{
+			RateLimiter:             reasonable.RateLimiter(),
+			MaxConcurrentReconciles: 10,
+		}).
+		Complete(reconcile.AsReconciler(m.GetClient(), c))
+}
+
+// Updating `ecsnodeclass-hash-version` annotation inside the karpenter controller means a breaking change has been made to the hash calculation.
+// `ecsnodeclass-hash` annotation on the ecsNodeClass will be updated, due to the breaking change, making the `ecsnodeclass-hash` on the NodeClaim different from
+// ecsNodeClass. Since, we cannot rely on the `ecsnodeclass-hash` on the NodeClaims, due to the breaking change, we will need to re-calculate the hash and update the annotation.
+// For more information on the Drift Hash Versioning: https://github.com/kubernetes-sigs/karpenter/blob/main/designs/drift-hash-versioning.md
+func (c *Controller) updateNodeClaimHash(ctx context.Context, nodeClass *v1alpha1.ECSNodeClass) error {
+	ncList := &karpv1.NodeClaimList{}
+	if err := c.kubeClient.List(ctx, ncList, client.MatchingFields{"spec.nodeClassRef.name": nodeClass.Name}); err != nil {
+		return err
+	}
+
+	errs := make([]error, len(ncList.Items))
+	for i := range ncList.Items {
+		nc := ncList.Items[i]
+		stored := nc.DeepCopy()
+
+		if nc.Annotations[v1alpha1.AnnotationECSNodeClassHashVersion] != v1alpha1.ECSNodeClassHashVersion {
+			nc.Annotations = lo.Assign(nc.Annotations, map[string]string{
+				v1alpha1.AnnotationECSNodeClassHashVersion: v1alpha1.ECSNodeClassHashVersion,
+			})
+
+			// Any NodeClaim that is already drifted will remain drifted if the karpenter.k8s.aws/nodepool-hash-version doesn't match
+			// Since the hashing mechanism has changed we will not be able to determine if the drifted status of the NodeClaim has changed
+			if nc.StatusConditions().Get(karpv1.ConditionTypeDrifted) == nil {
+				nc.Annotations = lo.Assign(nc.Annotations, map[string]string{
+					v1alpha1.AnnotationECSNodeClassHash: nodeClass.Hash(),
+				})
+			}
+
+			if !equality.Semantic.DeepEqual(stored, nc) {
+				if err := c.kubeClient.Patch(ctx, &nc, client.MergeFrom(stored)); err != nil {
+					errs[i] = client.IgnoreNotFound(err)
+				}
+			}
+		}
+	}
+
+	return multierr.Combine(errs...)
+}