@@ -21,15 +21,19 @@ import (
2121 "fmt"
2222 "os"
2323 "path/filepath"
24+ "time"
2425
2526 . "github.com/onsi/ginkgo/v2"
2627 . "github.com/onsi/gomega"
2728 corev1 "k8s.io/api/core/v1"
2829 "k8s.io/utils/ptr"
30+ "sigs.k8s.io/controller-runtime/pkg/client"
2931
32+ clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
3033 "sigs.k8s.io/cluster-api/test/framework"
3134 "sigs.k8s.io/cluster-api/test/framework/clusterctl"
3235 "sigs.k8s.io/cluster-api/util"
36+ "sigs.k8s.io/cluster-api/util/patch"
3337)
3438
3539// MachineDeploymentRolloutSpecInput is the input for MachineDeploymentRolloutSpec.
@@ -135,6 +139,147 @@ func MachineDeploymentRolloutSpec(ctx context.Context, inputGetter func() Machin
135139 Namespace : clusterResources .Cluster .Namespace ,
136140 })
137141
142+ // Get all machines before doing in-place changes so we can check at the end that no machines were replaced.
143+ machinesBeforeInPlaceChanges := getMachinesByCluster (ctx , input .BootstrapClusterProxy .GetClient (), clusterResources .Cluster )
144+
145+ preExistingAlwaysTaint := clusterv1.MachineTaint {
146+ Key : "pre-existing-always-taint" ,
147+ Value : "always-value" ,
148+ Effect : corev1 .TaintEffectPreferNoSchedule ,
149+ Propagation : clusterv1 .MachineTaintPropagationAlways ,
150+ }
151+
152+ preExistingOnInitializationTaint := clusterv1.MachineTaint {
153+ Key : "pre-existing-on-initialization-taint" ,
154+ Value : "on-initialization-value" ,
155+ Effect : corev1 .TaintEffectPreferNoSchedule ,
156+ Propagation : clusterv1 .MachineTaintPropagationOnInitialization ,
157+ }
158+ addingAlwaysTaint := clusterv1.MachineTaint {
159+ Key : "added-always-taint" ,
160+ Value : "added-always-value" ,
161+ Effect : corev1 .TaintEffectPreferNoSchedule ,
162+ Propagation : clusterv1 .MachineTaintPropagationAlways ,
163+ }
164+
165+ addingOnInitializationTaint := clusterv1.MachineTaint {
166+ Key : "added-on-initialization-taint" ,
167+ Value : "added-on-initialization-value" ,
168+ Effect : corev1 .TaintEffectPreferNoSchedule ,
169+ Propagation : clusterv1 .MachineTaintPropagationOnInitialization ,
170+ }
171+
172+ wantMachineTaints := []clusterv1.MachineTaint {
173+ preExistingAlwaysTaint ,
174+ preExistingOnInitializationTaint ,
175+ }
176+ wantNodeTaints := toCoreV1Taints (
177+ preExistingAlwaysTaint ,
178+ preExistingOnInitializationTaint ,
179+ )
180+
181+ Byf ("Verify MachineDeployment Machines and Nodes have the correct taints" )
182+ wlClient := input .BootstrapClusterProxy .GetWorkloadCluster (ctx , clusterResources .Cluster .Namespace , clusterResources .Cluster .Name ).GetClient ()
183+ verifyMachineAndNodeTaints (ctx , verifyMachineAndNodeTaintsInput {
184+ BootstrapClusterClient : input .BootstrapClusterProxy .GetClient (),
185+ WorkloadClusterClient : wlClient ,
186+ ClusterName : clusterResources .Cluster .Name ,
187+ MachineDeployments : clusterResources .MachineDeployments ,
188+ MachineTaints : wantMachineTaints ,
189+ NodeTaints : wantNodeTaints ,
190+ })
191+
192+ Byf ("Verify in-place propagation by adding new taints to the MachineDeployment" )
193+ wantMachineTaints = []clusterv1.MachineTaint {
194+ preExistingAlwaysTaint ,
195+ preExistingOnInitializationTaint ,
196+ addingAlwaysTaint ,
197+ addingOnInitializationTaint ,
198+ }
199+ wantNodeTaints = toCoreV1Taints (
200+ preExistingAlwaysTaint ,
201+ preExistingOnInitializationTaint ,
202+ addingAlwaysTaint ,
203+ )
204+ for _ , md := range clusterResources .MachineDeployments {
205+ patchHelper , err := patch .NewHelper (md , input .BootstrapClusterProxy .GetClient ())
206+ Expect (err ).ToNot (HaveOccurred ())
207+ md .Spec .Template .Spec .Taints = wantMachineTaints
208+ Expect (patchHelper .Patch (ctx , md )).To (Succeed ())
209+ }
210+
211+ verifyMachineAndNodeTaints (ctx , verifyMachineAndNodeTaintsInput {
212+ BootstrapClusterClient : input .BootstrapClusterProxy .GetClient (),
213+ WorkloadClusterClient : wlClient ,
214+ ClusterName : clusterResources .Cluster .Name ,
215+ MachineDeployments : clusterResources .MachineDeployments ,
216+ MachineTaints : wantMachineTaints ,
217+ NodeTaints : wantNodeTaints ,
218+ })
219+
220+ Byf ("Verify in-place propagation when removing preExisting Always and OnInitialization taint from the nodes" )
221+ nodes := corev1.NodeList {}
222+ Expect (wlClient .List (ctx , & nodes )).To (Succeed ())
223+ // Remove the initial taints from the nodes.
224+ for _ , node := range nodes .Items {
225+ patchHelper , err := patch .NewHelper (& node , wlClient )
226+ Expect (err ).ToNot (HaveOccurred ())
227+ newTaints := []corev1.Taint {}
228+ for _ , taint := range node .Spec .Taints {
229+ if taint .Key == preExistingAlwaysTaint .Key {
230+ continue
231+ }
232+ if taint .Key == preExistingOnInitializationTaint .Key {
233+ continue
234+ }
235+ newTaints = append (newTaints , taint )
236+ }
237+ node .Spec .Taints = newTaints
238+ Expect (patchHelper .Patch (ctx , & node )).To (Succeed ())
239+ }
240+
241+ wantNodeTaints = toCoreV1Taints (
242+ preExistingAlwaysTaint ,
243+ addingAlwaysTaint ,
244+ )
245+
246+ verifyMachineAndNodeTaints (ctx , verifyMachineAndNodeTaintsInput {
247+ BootstrapClusterClient : input .BootstrapClusterProxy .GetClient (),
248+ WorkloadClusterClient : wlClient ,
249+ ClusterName : clusterResources .Cluster .Name ,
250+ MachineDeployments : clusterResources .MachineDeployments ,
251+ MachineTaints : wantMachineTaints ,
252+ NodeTaints : wantNodeTaints ,
253+ })
254+
255+ Byf ("Verify in-place propagation by removing taints from the MachineDeployment" )
256+ wantMachineTaints = []clusterv1.MachineTaint {
257+ preExistingOnInitializationTaint ,
258+ addingOnInitializationTaint ,
259+ }
260+ wantNodeTaints = toCoreV1Taints ()
261+ for _ , md := range clusterResources .MachineDeployments {
262+ patchHelper , err := patch .NewHelper (md , input .BootstrapClusterProxy .GetClient ())
263+ Expect (err ).ToNot (HaveOccurred ())
264+ md .Spec .Template .Spec .Taints = wantMachineTaints
265+ Expect (patchHelper .Patch (ctx , md )).To (Succeed ())
266+ }
267+
268+ verifyMachineAndNodeTaints (ctx , verifyMachineAndNodeTaintsInput {
269+ BootstrapClusterClient : input .BootstrapClusterProxy .GetClient (),
270+ WorkloadClusterClient : wlClient ,
271+ ClusterName : clusterResources .Cluster .Name ,
272+ MachineDeployments : clusterResources .MachineDeployments ,
273+ MachineTaints : wantMachineTaints ,
274+ NodeTaints : wantNodeTaints ,
275+ })
276+
277+ By ("Verifying there are no unexpected rollouts through in-place changes" )
278+ Consistently (func (g Gomega ) {
279+ machinesAfterInPlaceChanges := getMachinesByCluster (ctx , input .BootstrapClusterProxy .GetClient (), clusterResources .Cluster )
280+ g .Expect (machinesAfterInPlaceChanges .Equal (machinesBeforeInPlaceChanges )).To (BeTrue (), "Machines must not be replaced through in-place rollout" )
281+ }, 30 * time .Second , 1 * time .Second ).Should (Succeed ())
282+
138283 By ("PASSED!" )
139284 })
140285
@@ -143,3 +288,52 @@ func MachineDeploymentRolloutSpec(ctx context.Context, inputGetter func() Machin
143288 framework .DumpSpecResourcesAndCleanup (ctx , specName , input .BootstrapClusterProxy , input .ClusterctlConfigPath , input .ArtifactFolder , namespace , cancelWatches , clusterResources .Cluster , input .E2EConfig .GetIntervals , input .SkipCleanup )
144289 })
145290}
291+
292+ type verifyMachineAndNodeTaintsInput struct {
293+ BootstrapClusterClient client.Client
294+ WorkloadClusterClient client.Client
295+ ClusterName string
296+ MachineDeployments []* clusterv1.MachineDeployment
297+ MachineTaints []clusterv1.MachineTaint
298+ NodeTaints []corev1.Taint
299+ }
300+
301+ func verifyMachineAndNodeTaints (ctx context.Context , input verifyMachineAndNodeTaintsInput ) {
302+ Expect (ctx ).NotTo (BeNil (), "ctx is required for verifyMachineAndNodeTaints" )
303+ Expect (input .BootstrapClusterClient ).ToNot (BeNil (), "Invalid argument. input.BootstrapClusterClient can't be nil when calling verifyMachineAndNodeTaints" )
304+ Expect (input .WorkloadClusterClient ).ToNot (BeNil (), "Invalid argument. input.WorkloadClusterClient can't be nil when calling verifyMachineAndNodeTaints" )
305+ Expect (input .ClusterName ).NotTo (BeEmpty (), "Invalid argument. input.ClusterName can't be empty when calling verifyMachineAndNodeTaints" )
306+ Expect (input .MachineDeployments ).NotTo (BeNil (), "Invalid argument. input.MachineDeployments can't be nil when calling verifyMachineAndNodeTaints" )
307+
308+ Eventually (func (g Gomega ) {
309+ for _ , md := range input .MachineDeployments {
310+ machines := framework .GetMachinesByMachineDeployments (ctx , framework.GetMachinesByMachineDeploymentsInput {
311+ Lister : input .BootstrapClusterClient ,
312+ ClusterName : input .ClusterName ,
313+ Namespace : md .Namespace ,
314+ MachineDeployment : * md ,
315+ })
316+ g .Expect (machines ).To (HaveLen (int (ptr .Deref (md .Spec .Replicas , 0 ))))
317+ for _ , machine := range machines {
318+ g .Expect (machine .Spec .Taints ).To (ConsistOf (input .MachineTaints ))
319+ g .Expect (machine .Status .NodeRef .IsDefined ()).To (BeTrue ())
320+
321+ node := & corev1.Node {}
322+ g .Expect (input .WorkloadClusterClient .Get (ctx , client.ObjectKey {Name : machine .Status .NodeRef .Name }, node )).To (Succeed ())
323+ g .Expect (node .Spec .Taints ).To (ConsistOf (input .NodeTaints ))
324+ }
325+ }
326+ }, "1m" ).Should (Succeed ())
327+ }
328+
329+ func toCoreV1Taints (machineTaints ... clusterv1.MachineTaint ) []corev1.Taint {
330+ taints := []corev1.Taint {}
331+ for _ , machineTaint := range machineTaints {
332+ taints = append (taints , corev1.Taint {
333+ Key : machineTaint .Key ,
334+ Value : machineTaint .Value ,
335+ Effect : machineTaint .Effect ,
336+ })
337+ }
338+ return taints
339+ }
0 commit comments