taints: add e2e via md-rollout test

Author: chrischdi

Makefile

@@ -603,6 +603,7 @@ generate-e2e-templates-v1.11: $(KUSTOMIZE)
 generate-e2e-templates-main: $(KUSTOMIZE)
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-md-remediation --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-md-remediation.yaml
+	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-md-taints --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-md-taints.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-kcp-remediation --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-kcp-remediation.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-kcp-adoption/step1 --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-kcp-adoption.yaml
 	echo "---" >> $(DOCKER_TEMPLATES)/main/cluster-template-kcp-adoption.yaml

test/e2e/config/docker.yaml

@@ -332,6 +332,7 @@ providers:
     # Add cluster templates
     - sourcePath: "../data/infrastructure-docker/main/cluster-template.yaml"
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-md-remediation.yaml"
+    - sourcePath: "../data/infrastructure-docker/main/cluster-template-md-taints.yaml"
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-kcp-remediation.yaml"
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-kcp-adoption.yaml"
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-machine-pool.yaml"
@@ -400,6 +401,7 @@ variables:
   EXP_MACHINE_SET_PREFLIGHT_CHECKS: "true"
   EXP_PRIORITY_QUEUE: "false"
   EXP_IN_PLACE_UPDATES: "true"
+  EXP_MACHINE_TAINT_PROPAGATION: "true"
   CAPI_DIAGNOSTICS_ADDRESS: ":8080"
   CAPI_INSECURE_DIAGNOSTICS: "true"
 

test/e2e/data/infrastructure-docker/main/cluster-template-md-taints/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+- ../bases/cluster-with-kcp.yaml
+- ../bases/md.yaml
+- ../bases/crs.yaml
+
+patches:
+- path: md-taints.yaml

test/e2e/data/infrastructure-docker/main/cluster-template-md-taints/md-taints.yaml

@@ -0,0 +1,16 @@
+apiVersion: cluster.x-k8s.io/v1beta2
+kind: MachineDeployment
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  template:
+    spec:
+      taints:
+        - key: "pre-existing-on-initialization-taint"
+          value: "on-initialization-value"
+          effect: PreferNoSchedule
+          propagation: OnInitialization
+        - key: "pre-existing-always-taint"
+          value: "always-value"
+          effect: PreferNoSchedule
+          propagation: Always

test/e2e/md_rollout.go

@@ -26,10 +26,13 @@ import (
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/utils/ptr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
 	"sigs.k8s.io/cluster-api/util"
+	"sigs.k8s.io/cluster-api/util/patch"
 )
 
 // MachineDeploymentRolloutSpecInput is the input for MachineDeploymentRolloutSpec.
@@ -135,6 +138,138 @@ func MachineDeploymentRolloutSpec(ctx context.Context, inputGetter func() Machin
 			Namespace: clusterResources.Cluster.Namespace,
 		})
 
+		preExistingAlwaysTaint := clusterv1.MachineTaint{
+			Key:         "pre-existing-always-taint",
+			Value:       "always-value",
+			Effect:      corev1.TaintEffectPreferNoSchedule,
+			Propagation: clusterv1.MachineTaintPropagationAlways,
+		}
+
+		preExistingOnInitializationTaint := clusterv1.MachineTaint{
+			Key:         "pre-existing-on-initialization-taint",
+			Value:       "on-initialization-value",
+			Effect:      corev1.TaintEffectPreferNoSchedule,
+			Propagation: clusterv1.MachineTaintPropagationOnInitialization,
+		}
+		addingAlwaysTaint := clusterv1.MachineTaint{
+			Key:         "added-always-taint",
+			Value:       "added-always-value",
+			Effect:      corev1.TaintEffectPreferNoSchedule,
+			Propagation: clusterv1.MachineTaintPropagationAlways,
+		}
+
+		addingOnInitializationTaint := clusterv1.MachineTaint{
+			Key:         "added-on-initialization-taint",
+			Value:       "added-on-initialization-value",
+			Effect:      corev1.TaintEffectPreferNoSchedule,
+			Propagation: clusterv1.MachineTaintPropagationOnInitialization,
+		}
+
+		wantMachineTaints := []clusterv1.MachineTaint{
+			preExistingAlwaysTaint,
+			preExistingOnInitializationTaint,
+		}
+		wantNodeTaints := toCoreV1Taints(
+			preExistingAlwaysTaint,
+			preExistingOnInitializationTaint,
+		)
+
+		Byf("Verify MachineDeployment Machines and Nodes have the correct taints")
+		wlClient := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, clusterResources.Cluster.Namespace, clusterResources.Cluster.Name).GetClient()
+		verifyMachineAndNodeTaints(ctx, verifyMachineAndNodeTaintsInput{
+			BootstrapClusterClient: input.BootstrapClusterProxy.GetClient(),
+			WorkloadClusterClient:  wlClient,
+			ClusterName:            clusterResources.Cluster.Name,
+			MachineDeployments:     clusterResources.MachineDeployments,
+			MachineTaints:          wantMachineTaints,
+			NodeTaints:             wantNodeTaints,
+		})
+
+		Byf("Verify in-place propagation by adding new taints to the MachineDeployment")
+		wantMachineTaints = []clusterv1.MachineTaint{
+			preExistingAlwaysTaint,
+			preExistingOnInitializationTaint,
+			addingAlwaysTaint,
+			addingOnInitializationTaint,
+		}
+		wantNodeTaints = toCoreV1Taints(
+			preExistingAlwaysTaint,
+			preExistingOnInitializationTaint,
+			addingAlwaysTaint,
+		)
+		for _, md := range clusterResources.MachineDeployments {
+			patchHelper, err := patch.NewHelper(md, input.BootstrapClusterProxy.GetClient())
+			Expect(err).ToNot(HaveOccurred())
+			md.Spec.Template.Spec.Taints = wantMachineTaints
+			Expect(patchHelper.Patch(ctx, md)).To(Succeed())
+		}
+
+		verifyMachineAndNodeTaints(ctx, verifyMachineAndNodeTaintsInput{
+			BootstrapClusterClient: input.BootstrapClusterProxy.GetClient(),
+			WorkloadClusterClient:  wlClient,
+			ClusterName:            clusterResources.Cluster.Name,
+			MachineDeployments:     clusterResources.MachineDeployments,
+			MachineTaints:          wantMachineTaints,
+			NodeTaints:             wantNodeTaints,
+		})
+
+		Byf("Verify in-place propagation when removing an Always taint from the nodes")
+		nodes := corev1.NodeList{}
+		Expect(wlClient.List(ctx, &nodes)).To(Succeed())
+		// Remove the initial taints from the nodes.
+		for _, node := range nodes.Items {
+			patchHelper, err := patch.NewHelper(&node, wlClient)
+			Expect(err).ToNot(HaveOccurred())
+			newTaints := []corev1.Taint{}
+			for _, taint := range node.Spec.Taints {
+				if taint.Key == preExistingAlwaysTaint.Key {
+					continue
+				}
+				if taint.Key == preExistingOnInitializationTaint.Key {
+					continue
+				}
+				newTaints = append(newTaints, taint)
+			}
+			node.Spec.Taints = newTaints
+			Expect(patchHelper.Patch(ctx, &node)).To(Succeed())
+		}
+
+		wantNodeTaints = toCoreV1Taints(
+			preExistingAlwaysTaint,
+			addingAlwaysTaint,
+		)
+
+		verifyMachineAndNodeTaints(ctx, verifyMachineAndNodeTaintsInput{
+			BootstrapClusterClient: input.BootstrapClusterProxy.GetClient(),
+			WorkloadClusterClient:  wlClient,
+			ClusterName:            clusterResources.Cluster.Name,
+			MachineDeployments:     clusterResources.MachineDeployments,
+			MachineTaints:          wantMachineTaints,
+			NodeTaints:             wantNodeTaints,
+		})
+
+		Byf("Verify in-place propagation by removing taints from the MachineDeployment")
+		wantMachineTaints = []clusterv1.MachineTaint{
+			preExistingOnInitializationTaint,
+			addingOnInitializationTaint,
+		}
+		wantNodeTaints = toCoreV1Taints()
+		for _, md := range clusterResources.MachineDeployments {
+			patchHelper, err := patch.NewHelper(md, input.BootstrapClusterProxy.GetClient())
+			Expect(err).ToNot(HaveOccurred())
+			md.Spec.Template.Spec.Taints = wantMachineTaints
+			Expect(patchHelper.Patch(ctx, md)).To(Succeed())
+		}
+
+		verifyMachineAndNodeTaints(ctx, verifyMachineAndNodeTaintsInput{
+			BootstrapClusterClient: input.BootstrapClusterProxy.GetClient(),
+			WorkloadClusterClient:  wlClient,
+			ClusterName:            clusterResources.Cluster.Name,
+			MachineDeployments:     clusterResources.MachineDeployments,
+			MachineTaints:          wantMachineTaints,
+			NodeTaints:             wantNodeTaints,
+		})
+
 		By("PASSED!")
 	})
 
@@ -143,3 +278,52 @@ func MachineDeploymentRolloutSpec(ctx context.Context, inputGetter func() Machin
 		framework.DumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ClusterctlConfigPath, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
 	})
 }
+
+type verifyMachineAndNodeTaintsInput struct {
+	BootstrapClusterClient client.Client
+	WorkloadClusterClient  client.Client
+	ClusterName            string
+	MachineDeployments     []*clusterv1.MachineDeployment
+	MachineTaints          []clusterv1.MachineTaint
+	NodeTaints             []corev1.Taint
+}
+
+func verifyMachineAndNodeTaints(ctx context.Context, input verifyMachineAndNodeTaintsInput) {
+	Expect(ctx).NotTo(BeNil(), "ctx is required for verifyMachineAndNodeTaints")
+	Expect(input.BootstrapClusterClient).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterClient can't be nil when calling verifyMachineAndNodeTaints")
+	Expect(input.WorkloadClusterClient).ToNot(BeNil(), "Invalid argument. input.WorkloadClusterClient can't be nil when calling verifyMachineAndNodeTaints")
+	Expect(input.ClusterName).NotTo(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling verifyMachineAndNodeTaints")
+	Expect(input.MachineDeployments).NotTo(BeNil(), "Invalid argument. input.MachineDeployments can't be nil when calling verifyMachineAndNodeTaints")
+
+	Eventually(func(g Gomega) {
+		for _, md := range input.MachineDeployments {
+			machines := framework.GetMachinesByMachineDeployments(ctx, framework.GetMachinesByMachineDeploymentsInput{
+				Lister:            input.BootstrapClusterClient,
+				ClusterName:       input.ClusterName,
+				Namespace:         md.Namespace,
+				MachineDeployment: *md,
+			})
+			g.Expect(machines).To(HaveLen(int(ptr.Deref(md.Spec.Replicas, 0))))
+			for _, machine := range machines {
+				g.Expect(machine.Spec.Taints).To(ConsistOf(input.MachineTaints))
+				g.Expect(machine.Status.NodeRef.IsDefined()).To(BeTrue())
+
+				node := &corev1.Node{}
+				g.Expect(input.WorkloadClusterClient.Get(ctx, client.ObjectKey{Name: machine.Status.NodeRef.Name}, node)).To(Succeed())
+				g.Expect(node.Spec.Taints).To(ConsistOf(input.NodeTaints))
+			}
+		}
+	}, "1m").Should(Succeed())
+}
+
+func toCoreV1Taints(machineTaints ...clusterv1.MachineTaint) []corev1.Taint {
+	taints := []corev1.Taint{}
+	for _, machineTaint := range machineTaints {
+		taints = append(taints, corev1.Taint{
+			Key:    machineTaint.Key,
+			Value:  machineTaint.Value,
+			Effect: machineTaint.Effect,
+		})
+	}
+	return taints
+}

test/e2e/md_rollout_test.go

@@ -30,6 +30,7 @@ var _ = Describe("When testing MachineDeployment rolling upgrades", func() {
 			ClusterctlConfigPath:  clusterctlConfigPath,
 			BootstrapClusterProxy: bootstrapClusterProxy,
 			ArtifactFolder:        artifactFolder,
+			Flavor:                "md-taints",
 			SkipCleanup:           skipCleanup,
 		}
 	})