From da1279f8ebdedebb61747457097e9da1996eb6b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Tue, 14 Oct 2025 22:22:03 +0200 Subject: [PATCH 01/17] feature: zalando.org/forward-backend annotation support to enable migration to eks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Usage: ``` apiVersion: zalando.org/v1 kind: StackSet metadata: annotations: zalando.org/forward-backend: eks migration .. ``` This will execute the migration preparation, such that the next traffic switch will send the traffic to the forward backend. Signed-off-by: Sandor Szücs --- pkg/core/stackset.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pkg/core/stackset.go b/pkg/core/stackset.go index 1fe5867e..f47330ac 100644 --- a/pkg/core/stackset.go +++ b/pkg/core/stackset.go @@ -17,6 +17,8 @@ const ( StackVersionLabelKey = "stack-version" ingressTrafficAuthoritativeAnnotation = "zalando.org/traffic-authoritative" + forwardBackendAnnotation = "zalando.org/forward-backend" + forwardBackendName = "fwd" ) var ( @@ -50,8 +52,25 @@ func sanitizeServicePorts(service *zv1.StackServiceSpec) *zv1.StackServiceSpec { return service } +func patchForwardBackend(rg *zv1.RouteGroupSpec) { + rg.AdditionalBackends = []rgv1.RouteGroupBackend{ + { + Name: forwardBackendName, + Type: rgv1.ForwardRouteGroupBackend, + }, + } + for _, route := range rg.Routes { + route.Backends = []rgv1.RouteGroupBackendReference{ + { + BackendName: forwardBackendName, + }, + } + } +} + // NewStack returns an (optional) stack that should be created func (ssc *StackSetContainer) NewStack() (*StackContainer, string) { + _, forwardMigration := ssc.StackSet.ObjectMeta.Annotations[forwardBackendAnnotation] observedStackVersion := ssc.StackSet.Status.ObservedStackVersion stackVersion := currentStackVersion(ssc.StackSet) stackName := generateStackName(ssc.StackSet, stackVersion) @@ -70,6 +89,9 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) { if ssc.StackSet.Spec.Ingress != nil { spec.Ingress = ssc.StackSet.Spec.Ingress.DeepCopy() + if forwardMigration { + spec.Ingress.EmbeddedObjectMetaWithAnnotations.Annotations["zalando.org/skipper-backend"] = "forward" + } } if ssc.StackSet.Spec.ExternalIngress != nil { @@ -78,6 +100,9 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) { if ssc.StackSet.Spec.RouteGroup != nil { spec.RouteGroup = ssc.StackSet.Spec.RouteGroup.DeepCopy() + if forwardMigration { + patchForwardBackend(spec.RouteGroup) + } } return &StackContainer{ From 96701d548dc863c076f03b70e5270c3a0c6ff0a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 24 Oct 2025 22:03:17 +0200 Subject: [PATCH 02/17] feature: externalIngress needs to be notified that we do a cluster migration, too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- docs/stack_crd.yaml | 16 ++++++++++++++ docs/stackset_crd.yaml | 22 ++++++++++++++----- pkg/apis/zalando.org/v1/types.go | 3 ++- .../zalando.org/v1/zz_generated.deepcopy.go | 5 +++-- 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/docs/stack_crd.yaml b/docs/stack_crd.yaml index deece1e6..801f6d5b 100644 --- a/docs/stack_crd.yaml +++ b/docs/stack_crd.yaml @@ -475,6 +475,22 @@ spec: - type: integer - type: string x-kubernetes-int-or-string: true + metadata: + description: |- + EmbeddedObjectMetaWithAnnotations defines the metadata which can be attached + to a resource. It's a slimmed down version of metav1.ObjectMeta only + containing annotations. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + type: object required: - backendPort type: object diff --git a/docs/stackset_crd.yaml b/docs/stackset_crd.yaml index 3213f79d..ea242252 100644 --- a/docs/stackset_crd.yaml +++ b/docs/stackset_crd.yaml @@ -67,6 +67,22 @@ spec: - type: integer - type: string x-kubernetes-int-or-string: true + metadata: + description: |- + EmbeddedObjectMetaWithAnnotations defines the metadata which can be attached + to a resource. It's a slimmed down version of metav1.ObjectMeta only + containing annotations. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + type: object required: - backendPort type: object @@ -3589,12 +3605,6 @@ spec: Must be set if and only if type is "Localhost". type: string type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. type: string required: - type diff --git a/pkg/apis/zalando.org/v1/types.go b/pkg/apis/zalando.org/v1/types.go index 74191701..8492ea3e 100644 --- a/pkg/apis/zalando.org/v1/types.go +++ b/pkg/apis/zalando.org/v1/types.go @@ -124,7 +124,8 @@ func (s *StackSetIngressSpec) GetAnnotations() map[string]string { // backendport for ingress managed outside of stackset. // +k8s:deepcopy-gen=true type StackSetExternalIngressSpec struct { - BackendPort intstr.IntOrString `json:"backendPort"` + EmbeddedObjectMetaWithAnnotations `json:"metadata,omitempty"` + BackendPort intstr.IntOrString `json:"backendPort"` } // RouteGroupSpec defines the specification for defining a RouteGroup attached diff --git a/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go b/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go index c55d3c03..fbc4a70c 100644 --- a/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go +++ b/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go @@ -745,6 +745,7 @@ func (in *StackSet) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StackSetExternalIngressSpec) DeepCopyInto(out *StackSetExternalIngressSpec) { *out = *in + in.EmbeddedObjectMetaWithAnnotations.DeepCopyInto(&out.EmbeddedObjectMetaWithAnnotations) out.BackendPort = in.BackendPort return } @@ -826,7 +827,7 @@ func (in *StackSetSpec) DeepCopyInto(out *StackSetSpec) { if in.ExternalIngress != nil { in, out := &in.ExternalIngress, &out.ExternalIngress *out = new(StackSetExternalIngressSpec) - **out = **in + (*in).DeepCopyInto(*out) } if in.RouteGroup != nil { in, out := &in.RouteGroup, &out.RouteGroup @@ -937,7 +938,7 @@ func (in *StackSpecInternal) DeepCopyInto(out *StackSpecInternal) { if in.ExternalIngress != nil { in, out := &in.ExternalIngress, &out.ExternalIngress *out = new(StackSetExternalIngressSpec) - **out = **in + (*in).DeepCopyInto(*out) } if in.Ingress != nil { in, out := &in.Ingress, &out.Ingress From 303f906722bb97b5fcebceabeb040748ddfe081c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 24 Oct 2025 22:08:27 +0200 Subject: [PATCH 03/17] refactor: move resource patching into the stack feature: reduce resource waste by setting replicas to 1 for the new unused stack, that will not receive any traffic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- pkg/core/stackset.go | 29 ++++-------------- pkg/core/types.go | 73 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 67 insertions(+), 35 deletions(-) diff --git a/pkg/core/stackset.go b/pkg/core/stackset.go index f47330ac..63eb8a59 100644 --- a/pkg/core/stackset.go +++ b/pkg/core/stackset.go @@ -52,22 +52,6 @@ func sanitizeServicePorts(service *zv1.StackServiceSpec) *zv1.StackServiceSpec { return service } -func patchForwardBackend(rg *zv1.RouteGroupSpec) { - rg.AdditionalBackends = []rgv1.RouteGroupBackend{ - { - Name: forwardBackendName, - Type: rgv1.ForwardRouteGroupBackend, - }, - } - for _, route := range rg.Routes { - route.Backends = []rgv1.RouteGroupBackendReference{ - { - BackendName: forwardBackendName, - }, - } - } -} - // NewStack returns an (optional) stack that should be created func (ssc *StackSetContainer) NewStack() (*StackContainer, string) { _, forwardMigration := ssc.StackSet.ObjectMeta.Annotations[forwardBackendAnnotation] @@ -89,9 +73,6 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) { if ssc.StackSet.Spec.Ingress != nil { spec.Ingress = ssc.StackSet.Spec.Ingress.DeepCopy() - if forwardMigration { - spec.Ingress.EmbeddedObjectMetaWithAnnotations.Annotations["zalando.org/skipper-backend"] = "forward" - } } if ssc.StackSet.Spec.ExternalIngress != nil { @@ -100,9 +81,11 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) { if ssc.StackSet.Spec.RouteGroup != nil { spec.RouteGroup = ssc.StackSet.Spec.RouteGroup.DeepCopy() - if forwardMigration { - patchForwardBackend(spec.RouteGroup) - } + } + + stackAnnotations := ssc.StackSet.Spec.StackTemplate.Annotations + if forwardMigration { + stackAnnotations[forwardBackendAnnotation] = forwardBackendName } return &StackContainer{ @@ -125,7 +108,7 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) { ssc.StackSet.Labels, map[string]string{StackVersionLabelKey: stackVersion}, ), - Annotations: ssc.StackSet.Spec.StackTemplate.Annotations, + Annotations: stackAnnotations, }, Spec: *spec, }, diff --git a/pkg/core/types.go b/pkg/core/types.go index 324a269f..654e8b26 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -419,7 +419,14 @@ func (sc *StackContainer) updateStackResources() error { } func (sc *StackContainer) updateFromResources() { - sc.stackReplicas = effectiveReplicas(sc.Stack.Spec.StackSpec.Replicas) + _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation] + + if clusterMigration { + // we do not use these so reduce wasted resources! + sc.stackReplicas = 1 + } else { + sc.stackReplicas = effectiveReplicas(sc.Stack.Spec.StackSpec.Replicas) + } var deploymentUpdated, serviceUpdated, ingressUpdated, routeGroupUpdated, hpaUpdated bool var ingressSegmentUpdated, routeGroupSegmentUpdated bool @@ -427,10 +434,19 @@ func (sc *StackContainer) updateFromResources() { // deployment if sc.Resources.Deployment != nil { deployment := sc.Resources.Deployment - sc.deploymentReplicas = effectiveReplicas(deployment.Spec.Replicas) - sc.createdReplicas = deployment.Status.Replicas - sc.readyReplicas = deployment.Status.ReadyReplicas - sc.updatedReplicas = deployment.Status.UpdatedReplicas + + if clusterMigration { + // we do not use these so reduce wasted resources! + sc.deploymentReplicas = 1 + sc.createdReplicas = 1 + sc.readyReplicas = 1 + sc.updatedReplicas = 1 + } else { + sc.deploymentReplicas = effectiveReplicas(deployment.Spec.Replicas) + sc.createdReplicas = deployment.Status.Replicas + sc.readyReplicas = deployment.Status.ReadyReplicas + sc.updatedReplicas = deployment.Status.UpdatedReplicas + } deploymentUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.Deployment.ObjectMeta) && deployment.Status.ObservedGeneration == deployment.Generation } @@ -442,13 +458,20 @@ func (sc *StackContainer) updateFromResources() { // the per-stack ingress must either be present and up-to-date, or not present and not expected. // the per-stack ingress is not expected if the stack has no hostnames matching the cluster domain. if sc.Resources.Ingress != nil { + if clusterMigration { + sc.Resources.Ingress.Annotations["zalando.org/skipper-backend"] = "forward" + } ingressUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.Ingress.ObjectMeta) } else { hostnames := sc.stackHostnames(sc.ingressSpec, false) ingressUpdated = len(hostnames) == 0 } - ingressSegmentUpdated = sc.Resources.IngressSegment != nil && - IsResourceUpToDate(sc.Stack, sc.Resources.IngressSegment.ObjectMeta) + if sc.Resources.IngressSegment != nil { + if clusterMigration { + sc.Resources.IngressSegment.Annotations["zalando.org/skipper-backend"] = "forward" + } + ingressSegmentUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.IngressSegment.ObjectMeta) + } } else { // ignore if ingress is not set ingressUpdated = sc.Resources.Ingress == nil @@ -460,24 +483,33 @@ func (sc *StackContainer) updateFromResources() { // the per-stack route group must either be present and up-to-date, or not present and not expected. // the per-stack route group is not expected if the stack has no hostnames matching the cluster domain. if sc.Resources.RouteGroup != nil { + if clusterMigration { + patchForwardBackend(&sc.Resources.RouteGroup.Spec) + } routeGroupUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.RouteGroup.ObjectMeta) } else { hostnames := sc.stackHostnames(sc.routeGroupSpec, false) routeGroupUpdated = len(hostnames) == 0 } - routeGroupSegmentUpdated = sc.Resources.RouteGroupSegment != nil && - IsResourceUpToDate( + + if sc.Resources.RouteGroupSegment != nil { + if clusterMigration { + patchForwardBackend(&sc.Resources.RouteGroupSegment.Spec) + } + + routeGroupSegmentUpdated = IsResourceUpToDate( sc.Stack, sc.Resources.RouteGroupSegment.ObjectMeta, ) + } } else { // ignore if route group is not set routeGroupUpdated = sc.Resources.RouteGroup == nil routeGroupSegmentUpdated = sc.Resources.RouteGroupSegment == nil } - // hpa - if sc.IsAutoscaled() { + // hpa not used if we migrate cluster to reduce wasted resources + if sc.IsAutoscaled() && !clusterMigration { hpaUpdated = sc.Resources.HPA != nil && IsResourceUpToDate(sc.Stack, sc.Resources.HPA.ObjectMeta) } else { hpaUpdated = sc.Resources.HPA == nil @@ -494,10 +526,27 @@ func (sc *StackContainer) updateFromResources() { status := sc.Stack.Status sc.noTrafficSince = unwrapTime(status.NoTrafficSince) - if status.Prescaling.Active { + // do not prescale on cluste rmigration to reduce wasted resources + if status.Prescaling.Active && !clusterMigration { sc.prescalingActive = true sc.prescalingReplicas = status.Prescaling.Replicas sc.prescalingDesiredTrafficWeight = status.Prescaling.DesiredTrafficWeight sc.prescalingLastTrafficIncrease = unwrapTime(status.Prescaling.LastTrafficIncrease) } } + +func patchForwardBackend(rg *rgv1.RouteGroupSpec) { + rg.Backends = []rgv1.RouteGroupBackend{ + { + Name: forwardBackendName, + Type: rgv1.ForwardRouteGroupBackend, + }, + } + for _, route := range rg.Routes { + route.Backends = []rgv1.RouteGroupBackendReference{ + { + BackendName: forwardBackendName, + }, + } + } +} From 77c2e0c8eb6e994cca2de9275b9a12607c973118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 24 Oct 2025 22:20:22 +0200 Subject: [PATCH 04/17] doc: add context how the migration will work so we will understand in the future why we scale down to 1 fix: ExternalIngress has also to get the anntoation to be set, such that it can execute a migration as expected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- pkg/core/types.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pkg/core/types.go b/pkg/core/types.go index 654e8b26..d53ccd07 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -418,11 +418,20 @@ func (sc *StackContainer) updateStackResources() error { return nil } +// updateFromResources updates stack from all the containing +// resources. On cluster migrations we do traffic switching to the +// other cluster, so all the new pods from this stack will not receive +// any traffic. Therefore we can set replicas to 1 and remove hpa +// whatever was configured. Ingress and ExternaIngress will have an +// annotation set that it will send traffic to the other +// cluster. RouteGroup will get a patched backend, such that it does +// the same as ingress. func (sc *StackContainer) updateFromResources() { - _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation] + fwdVal, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation] if clusterMigration { - // we do not use these so reduce wasted resources! + sc.Stack.Spec.ExternalIngress.Annotations[forwardBackendAnnotation] = fwdVal + // we do not use these pods so reduce wasted resources! sc.stackReplicas = 1 } else { sc.stackReplicas = effectiveReplicas(sc.Stack.Spec.StackSpec.Replicas) @@ -436,7 +445,7 @@ func (sc *StackContainer) updateFromResources() { deployment := sc.Resources.Deployment if clusterMigration { - // we do not use these so reduce wasted resources! + // we do not use these pods so reduce wasted resources! sc.deploymentReplicas = 1 sc.createdReplicas = 1 sc.readyReplicas = 1 @@ -526,7 +535,7 @@ func (sc *StackContainer) updateFromResources() { status := sc.Stack.Status sc.noTrafficSince = unwrapTime(status.NoTrafficSince) - // do not prescale on cluste rmigration to reduce wasted resources + // do not prescale on cluster migration to reduce wasted resources if status.Prescaling.Active && !clusterMigration { sc.prescalingActive = true sc.prescalingReplicas = status.Prescaling.Replicas From ae8b4589059f0e67623cb27c9ed5aab424c1f826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Thu, 6 Nov 2025 19:55:03 +0100 Subject: [PATCH 05/17] reafctor: change to pathc all resource creations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- pkg/core/stack_resources.go | 36 +++++++++++++++++++++ pkg/core/stackset.go | 3 ++ pkg/core/types.go | 64 ++++++------------------------------- 3 files changed, 48 insertions(+), 55 deletions(-) diff --git a/pkg/core/stack_resources.go b/pkg/core/stack_resources.go index aee61911..b322716c 100644 --- a/pkg/core/stack_resources.go +++ b/pkg/core/stack_resources.go @@ -76,6 +76,23 @@ func objectMetaInjectLabels(objectMeta metav1.ObjectMeta, labels map[string]stri return objectMeta } +// patchForwardBackend rewrites a RouteGroupSpec to send all traffic to another cluster chosen by the operator of skipper +func patchForwardBackend(rg *rgv1.RouteGroupSpec) { + rg.Backends = []rgv1.RouteGroupBackend{ + { + Name: forwardBackendName, + Type: rgv1.ForwardRouteGroupBackend, + }, + } + for _, route := range rg.Routes { + route.Backends = []rgv1.RouteGroupBackendReference{ + { + BackendName: forwardBackendName, + }, + } + } +} + func (sc *StackContainer) objectMeta(segment bool) metav1.ObjectMeta { resourceLabels := mapCopy(sc.Stack.Labels) @@ -224,6 +241,12 @@ func (sc *StackContainer) GenerateDeployment() *appsv1.Deployment { if strategy != nil { deployment.Spec.Strategy = *strategy } + + if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration { + i := int32(1) + deployment.Spec.Replicas = &i + } + return deployment } @@ -281,6 +304,12 @@ func (sc *StackContainer) GenerateHPA() ( result.Spec.MinReplicas = &pr } + if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration { + i := int32(1) + result.Spec.MinReplicas = &i + result.Spec.MaxReplicas = i + } + return result, nil } @@ -430,6 +459,9 @@ func (sc *StackContainer) generateIngress(segment bool) ( Rules: rules, }, } + if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration { + result.Annotations["zalando.org/skipper-backend"] = "forward" + } // insert annotations result.Annotations = mergeLabels( @@ -529,6 +561,10 @@ func (sc *StackContainer) generateRouteGroup(segment bool) ( sc.routeGroupSpec.GetAnnotations(), ) + if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration { + patchForwardBackend(&result.Spec) + } + return result, nil } diff --git a/pkg/core/stackset.go b/pkg/core/stackset.go index 63eb8a59..94b659a2 100644 --- a/pkg/core/stackset.go +++ b/pkg/core/stackset.go @@ -77,6 +77,9 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) { if ssc.StackSet.Spec.ExternalIngress != nil { spec.ExternalIngress = ssc.StackSet.Spec.ExternalIngress.DeepCopy() + if forwardMigration { + spec.ExternalIngress.Annotations[forwardBackendAnnotation] = forwardBackendName + } } if ssc.StackSet.Spec.RouteGroup != nil { diff --git a/pkg/core/types.go b/pkg/core/types.go index d53ccd07..7b05df11 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -427,15 +427,7 @@ func (sc *StackContainer) updateStackResources() error { // cluster. RouteGroup will get a patched backend, such that it does // the same as ingress. func (sc *StackContainer) updateFromResources() { - fwdVal, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation] - - if clusterMigration { - sc.Stack.Spec.ExternalIngress.Annotations[forwardBackendAnnotation] = fwdVal - // we do not use these pods so reduce wasted resources! - sc.stackReplicas = 1 - } else { - sc.stackReplicas = effectiveReplicas(sc.Stack.Spec.StackSpec.Replicas) - } + sc.stackReplicas = effectiveReplicas(sc.Stack.Spec.StackSpec.Replicas) var deploymentUpdated, serviceUpdated, ingressUpdated, routeGroupUpdated, hpaUpdated bool var ingressSegmentUpdated, routeGroupSegmentUpdated bool @@ -444,18 +436,11 @@ func (sc *StackContainer) updateFromResources() { if sc.Resources.Deployment != nil { deployment := sc.Resources.Deployment - if clusterMigration { - // we do not use these pods so reduce wasted resources! - sc.deploymentReplicas = 1 - sc.createdReplicas = 1 - sc.readyReplicas = 1 - sc.updatedReplicas = 1 - } else { - sc.deploymentReplicas = effectiveReplicas(deployment.Spec.Replicas) - sc.createdReplicas = deployment.Status.Replicas - sc.readyReplicas = deployment.Status.ReadyReplicas - sc.updatedReplicas = deployment.Status.UpdatedReplicas - } + sc.deploymentReplicas = effectiveReplicas(deployment.Spec.Replicas) + sc.createdReplicas = deployment.Status.Replicas + sc.readyReplicas = deployment.Status.ReadyReplicas + sc.updatedReplicas = deployment.Status.UpdatedReplicas + deploymentUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.Deployment.ObjectMeta) && deployment.Status.ObservedGeneration == deployment.Generation } @@ -467,18 +452,12 @@ func (sc *StackContainer) updateFromResources() { // the per-stack ingress must either be present and up-to-date, or not present and not expected. // the per-stack ingress is not expected if the stack has no hostnames matching the cluster domain. if sc.Resources.Ingress != nil { - if clusterMigration { - sc.Resources.Ingress.Annotations["zalando.org/skipper-backend"] = "forward" - } ingressUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.Ingress.ObjectMeta) } else { hostnames := sc.stackHostnames(sc.ingressSpec, false) ingressUpdated = len(hostnames) == 0 } if sc.Resources.IngressSegment != nil { - if clusterMigration { - sc.Resources.IngressSegment.Annotations["zalando.org/skipper-backend"] = "forward" - } ingressSegmentUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.IngressSegment.ObjectMeta) } } else { @@ -492,9 +471,6 @@ func (sc *StackContainer) updateFromResources() { // the per-stack route group must either be present and up-to-date, or not present and not expected. // the per-stack route group is not expected if the stack has no hostnames matching the cluster domain. if sc.Resources.RouteGroup != nil { - if clusterMigration { - patchForwardBackend(&sc.Resources.RouteGroup.Spec) - } routeGroupUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.RouteGroup.ObjectMeta) } else { hostnames := sc.stackHostnames(sc.routeGroupSpec, false) @@ -502,10 +478,6 @@ func (sc *StackContainer) updateFromResources() { } if sc.Resources.RouteGroupSegment != nil { - if clusterMigration { - patchForwardBackend(&sc.Resources.RouteGroupSegment.Spec) - } - routeGroupSegmentUpdated = IsResourceUpToDate( sc.Stack, sc.Resources.RouteGroupSegment.ObjectMeta, @@ -518,10 +490,8 @@ func (sc *StackContainer) updateFromResources() { } // hpa not used if we migrate cluster to reduce wasted resources - if sc.IsAutoscaled() && !clusterMigration { + if sc.IsAutoscaled() { hpaUpdated = sc.Resources.HPA != nil && IsResourceUpToDate(sc.Stack, sc.Resources.HPA.ObjectMeta) - } else { - hpaUpdated = sc.Resources.HPA == nil } // aggregated 'resources updated' for the readiness @@ -535,27 +505,11 @@ func (sc *StackContainer) updateFromResources() { status := sc.Stack.Status sc.noTrafficSince = unwrapTime(status.NoTrafficSince) - // do not prescale on cluster migration to reduce wasted resources - if status.Prescaling.Active && !clusterMigration { + + if status.Prescaling.Active { sc.prescalingActive = true sc.prescalingReplicas = status.Prescaling.Replicas sc.prescalingDesiredTrafficWeight = status.Prescaling.DesiredTrafficWeight sc.prescalingLastTrafficIncrease = unwrapTime(status.Prescaling.LastTrafficIncrease) } } - -func patchForwardBackend(rg *rgv1.RouteGroupSpec) { - rg.Backends = []rgv1.RouteGroupBackend{ - { - Name: forwardBackendName, - Type: rgv1.ForwardRouteGroupBackend, - }, - } - for _, route := range rg.Routes { - route.Backends = []rgv1.RouteGroupBackendReference{ - { - BackendName: forwardBackendName, - }, - } - } -} From c10449f4d229966de36af0839531a7394a649da5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Thu, 6 Nov 2025 20:09:11 +0100 Subject: [PATCH 06/17] doc: move the migration docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- pkg/core/stack_resources.go | 16 ++++++++++++++++ pkg/core/types.go | 10 ---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/pkg/core/stack_resources.go b/pkg/core/stack_resources.go index b322716c..7aa5a314 100644 --- a/pkg/core/stack_resources.go +++ b/pkg/core/stack_resources.go @@ -186,6 +186,10 @@ func (sc *StackContainer) selector() map[string]string { return limitLabels(sc.Stack.Labels, selectorLabels) } +// GenerateDeployment generates a deployment as configured in the +// stack. On cluster migrations set by stackset annotation +// "zalando.org/forward-backend", the deployment will be set to +// replicas 1. func (sc *StackContainer) GenerateDeployment() *appsv1.Deployment { stack := sc.Stack @@ -250,6 +254,10 @@ func (sc *StackContainer) GenerateDeployment() *appsv1.Deployment { return deployment } +// GenerateHPA generates a hpa as configured in the +// stack. On cluster migrations set by stackset annotation +// "zalando.org/forward-backend", the hpa will be set to +// minReplicas = maxReplicass = 1. func (sc *StackContainer) GenerateHPA() ( *autoscaling.HorizontalPodAutoscaler, error, @@ -408,6 +416,10 @@ func (sc *StackContainer) GenerateIngressSegment() ( return res, nil } +// generateIngress generates an ingress as configured in the stack. +// On cluster migrations set by stackset annotation +// "zalando.org/forward-backend", the annotation will be copied to the +// ingress. func (sc *StackContainer) generateIngress(segment bool) ( *networking.Ingress, error, @@ -502,6 +514,10 @@ func (sc *StackContainer) GenerateRouteGroupSegment() ( return res, nil } +// generateRouteGroup generates an RouteGroup as configured in the +// stack. On cluster migrations set by stackset annotation +// "zalando.org/forward-backend", the RouteGroup will be patched by +// patchForwardBackend() to execute the migration. func (sc *StackContainer) generateRouteGroup(segment bool) ( *rgv1.RouteGroup, error, diff --git a/pkg/core/types.go b/pkg/core/types.go index 7b05df11..5db6fa46 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -418,14 +418,6 @@ func (sc *StackContainer) updateStackResources() error { return nil } -// updateFromResources updates stack from all the containing -// resources. On cluster migrations we do traffic switching to the -// other cluster, so all the new pods from this stack will not receive -// any traffic. Therefore we can set replicas to 1 and remove hpa -// whatever was configured. Ingress and ExternaIngress will have an -// annotation set that it will send traffic to the other -// cluster. RouteGroup will get a patched backend, such that it does -// the same as ingress. func (sc *StackContainer) updateFromResources() { sc.stackReplicas = effectiveReplicas(sc.Stack.Spec.StackSpec.Replicas) @@ -435,12 +427,10 @@ func (sc *StackContainer) updateFromResources() { // deployment if sc.Resources.Deployment != nil { deployment := sc.Resources.Deployment - sc.deploymentReplicas = effectiveReplicas(deployment.Spec.Replicas) sc.createdReplicas = deployment.Status.Replicas sc.readyReplicas = deployment.Status.ReadyReplicas sc.updatedReplicas = deployment.Status.UpdatedReplicas - deploymentUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.Deployment.ObjectMeta) && deployment.Status.ObservedGeneration == deployment.Generation } From ce65958dc10e3f83f63de37d71058ac67ffb8fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Thu, 6 Nov 2025 20:13:36 +0100 Subject: [PATCH 07/17] revert pkg/core/types.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- pkg/core/types.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pkg/core/types.go b/pkg/core/types.go index 5db6fa46..324a269f 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -447,9 +447,8 @@ func (sc *StackContainer) updateFromResources() { hostnames := sc.stackHostnames(sc.ingressSpec, false) ingressUpdated = len(hostnames) == 0 } - if sc.Resources.IngressSegment != nil { - ingressSegmentUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.IngressSegment.ObjectMeta) - } + ingressSegmentUpdated = sc.Resources.IngressSegment != nil && + IsResourceUpToDate(sc.Stack, sc.Resources.IngressSegment.ObjectMeta) } else { // ignore if ingress is not set ingressUpdated = sc.Resources.Ingress == nil @@ -466,22 +465,22 @@ func (sc *StackContainer) updateFromResources() { hostnames := sc.stackHostnames(sc.routeGroupSpec, false) routeGroupUpdated = len(hostnames) == 0 } - - if sc.Resources.RouteGroupSegment != nil { - routeGroupSegmentUpdated = IsResourceUpToDate( + routeGroupSegmentUpdated = sc.Resources.RouteGroupSegment != nil && + IsResourceUpToDate( sc.Stack, sc.Resources.RouteGroupSegment.ObjectMeta, ) - } } else { // ignore if route group is not set routeGroupUpdated = sc.Resources.RouteGroup == nil routeGroupSegmentUpdated = sc.Resources.RouteGroupSegment == nil } - // hpa not used if we migrate cluster to reduce wasted resources + // hpa if sc.IsAutoscaled() { hpaUpdated = sc.Resources.HPA != nil && IsResourceUpToDate(sc.Stack, sc.Resources.HPA.ObjectMeta) + } else { + hpaUpdated = sc.Resources.HPA == nil } // aggregated 'resources updated' for the readiness @@ -495,7 +494,6 @@ func (sc *StackContainer) updateFromResources() { status := sc.Stack.Status sc.noTrafficSince = unwrapTime(status.NoTrafficSince) - if status.Prescaling.Active { sc.prescalingActive = true sc.prescalingReplicas = status.Prescaling.Replicas From ac011d795f2a27521b28db73eb79ae7a42a5ab62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 7 Nov 2025 09:32:30 +0100 Subject: [PATCH 08/17] refactor: simplify test check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- pkg/core/stack_resources_test.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pkg/core/stack_resources_test.go b/pkg/core/stack_resources_test.go index 80d2bbba..43187d77 100644 --- a/pkg/core/stack_resources_test.go +++ b/pkg/core/stack_resources_test.go @@ -830,15 +830,7 @@ func TestStackGenerateRouteGroupSegment(t *testing.T) { } for _, r := range rg.Spec.Routes { - found := false - for _, p := range r.Predicates { - if p == tc.expectedPredicate { - found = true - break - } - } - - if !found { + if !slices.Contains(r.Predicates, tc.expectedPredicate) { t.Errorf("predicate %q not found in route %v", tc.expectedPredicate, r, From 6a92b86b4ccf0b28f82d0b73bc4df9bab05b5636 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 7 Nov 2025 10:40:12 +0100 Subject: [PATCH 09/17] external ingress controllers should read stack.Annotations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- pkg/core/stackset.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/core/stackset.go b/pkg/core/stackset.go index 94b659a2..63eb8a59 100644 --- a/pkg/core/stackset.go +++ b/pkg/core/stackset.go @@ -77,9 +77,6 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) { if ssc.StackSet.Spec.ExternalIngress != nil { spec.ExternalIngress = ssc.StackSet.Spec.ExternalIngress.DeepCopy() - if forwardMigration { - spec.ExternalIngress.Annotations[forwardBackendAnnotation] = forwardBackendName - } } if ssc.StackSet.Spec.RouteGroup != nil { From 2647d92fb6dca1e9aedfc51844d303de15bf7079 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 7 Nov 2025 12:31:13 +0100 Subject: [PATCH 10/17] deployment test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- pkg/core/stack_resources_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkg/core/stack_resources_test.go b/pkg/core/stack_resources_test.go index 43187d77..5af65de7 100644 --- a/pkg/core/stack_resources_test.go +++ b/pkg/core/stack_resources_test.go @@ -2,6 +2,7 @@ package core import ( "errors" + "maps" "reflect" "slices" "strconv" @@ -1095,6 +1096,7 @@ func TestStackGenerateDeployment(t *testing.T) { expectedReplicas int32 maxUnavailable int maxSurge int + stackAnnotations map[string]string }{ { name: "stack scaled down to zero, deployment still running", @@ -1228,6 +1230,15 @@ func TestStackGenerateDeployment(t *testing.T) { name: "minReadySeconds should be set", minReadySeconds: 5, }, + { + name: "cluster migration should scale down deployment", + stackReplicas: 3, + deploymentReplicas: 3, + expectedReplicas: 1, + stackAnnotations: map[string]string{ + forwardBackendAnnotation: "fwd1", + }, + }, } { t.Run(tc.name, func(t *testing.T) { var strategy *apps.DeploymentStrategy @@ -1280,6 +1291,13 @@ func TestStackGenerateDeployment(t *testing.T) { if tc.hpaEnabled { c.Stack.Spec.StackSpec.Autoscaler = &zv1.Autoscaler{} } + if tc.stackAnnotations != nil { + if c.Stack.Annotations != nil { + maps.Copy(c.Stack.Annotations, tc.stackAnnotations) + } else { + c.Stack.Annotations = tc.stackAnnotations + } + } deployment := c.GenerateDeployment() expected := &apps.Deployment{ ObjectMeta: testResourceMeta, From c4a3a44754e13403ce291e096114b8dc833fac5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 7 Nov 2025 13:15:47 +0100 Subject: [PATCH 11/17] doc: link to ingress annotation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- pkg/core/stack_resources.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/core/stack_resources.go b/pkg/core/stack_resources.go index 7aa5a314..335481b3 100644 --- a/pkg/core/stack_resources.go +++ b/pkg/core/stack_resources.go @@ -472,6 +472,7 @@ func (sc *StackContainer) generateIngress(segment bool) ( }, } if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration { + // see https://opensource.zalando.com/skipper/kubernetes/ingress-usage/#skipper-ingress-annotations result.Annotations["zalando.org/skipper-backend"] = "forward" } From 97032a1c0439172a93ce8a44ab529279af222ece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 7 Nov 2025 13:16:16 +0100 Subject: [PATCH 12/17] add tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- pkg/core/stack_resources_test.go | 123 ++++++++++++++++++++++++++++++- 1 file changed, 120 insertions(+), 3 deletions(-) diff --git a/pkg/core/stack_resources_test.go b/pkg/core/stack_resources_test.go index 5af65de7..b3714b1d 100644 --- a/pkg/core/stack_resources_test.go +++ b/pkg/core/stack_resources_test.go @@ -282,8 +282,9 @@ func TestLimitLabels(t *testing.T) { func TestStackGenerateIngress(t *testing.T) { for _, tc := range []struct { - name string - ingressSpec *zv1.StackSetIngressSpec + name string + ingressSpec *zv1.StackSetIngressSpec + stackAnnotations map[string]string expectDisabled bool expectError bool @@ -310,6 +311,25 @@ func TestStackGenerateIngress(t *testing.T) { }, expectedHosts: []string{"foo-v1.example.org"}, }, + { + name: "cluster migration", + ingressSpec: &zv1.StackSetIngressSpec{ + EmbeddedObjectMetaWithAnnotations: zv1.EmbeddedObjectMetaWithAnnotations{ + Annotations: map[string]string{"ingress": "annotation"}, + }, + Hosts: []string{"foo.example.org", "foo.example.com"}, + Path: "example", + }, + stackAnnotations: map[string]string{ + forwardBackendAnnotation: "fwd-ingress", + }, + expectedAnnotations: map[string]string{ + stackGenerationAnnotationKey: "11", + "ingress": "annotation", + "zalando.org/skipper-backend": "forward", + }, + expectedHosts: []string{"foo-v1.example.org"}, + }, } { t.Run(tc.name, func(t *testing.T) { backendPort := int32(80) @@ -323,6 +343,10 @@ func TestStackGenerateIngress(t *testing.T) { backendPort: &intStrBackendPort, clusterDomains: []string{"example.org"}, } + if tc.stackAnnotations != nil { + c.Stack.Annotations = tc.stackAnnotations + } + ingress, err := c.GenerateIngress() if tc.expectError { @@ -592,10 +616,12 @@ func TestStackGenerateRouteGroup(t *testing.T) { for _, tc := range []struct { name string routeGroupSpec *zv1.RouteGroupSpec + stackAnnotations map[string]string expectDisabled bool expectError bool expectedAnnotations map[string]string expectedHosts []string + expectedBackend []rgv1.RouteGroupBackend }{ { name: "no route group spec", @@ -638,6 +664,34 @@ func TestStackGenerateRouteGroup(t *testing.T) { }, expectedHosts: []string{"foo-v1.example.org"}, }, + { + name: "cluster migration", + routeGroupSpec: &zv1.RouteGroupSpec{ + EmbeddedObjectMetaWithAnnotations: zv1.EmbeddedObjectMetaWithAnnotations{ + Annotations: map[string]string{"routegroup": "annotation"}, + }, + Hosts: []string{"foo.example.org", "foo.example.com"}, + Routes: []rgv1.RouteGroupRouteSpec{ + { + PathSubtree: "/example", + }, + }, + }, + stackAnnotations: map[string]string{ + forwardBackendAnnotation: "fwd-routegroup", + }, + expectedAnnotations: map[string]string{ + stackGenerationAnnotationKey: "11", + "routegroup": "annotation", + }, + expectedHosts: []string{"foo-v1.example.org"}, + expectedBackend: []rgv1.RouteGroupBackend{ + { + Name: "fwd", + Type: rgv1.ForwardRouteGroupBackend, + }, + }, + }, } { t.Run(tc.name, func(t *testing.T) { backendPort := int32(80) @@ -651,6 +705,13 @@ func TestStackGenerateRouteGroup(t *testing.T) { backendPort: &intStrBackendPort, clusterDomains: []string{"example.org"}, } + if tc.stackAnnotations != nil { + if c.Stack.Annotations != nil { + maps.Copy(c.Stack.Annotations, tc.stackAnnotations) + } else { + c.Stack.Annotations = tc.stackAnnotations + } + } rg, err := c.GenerateRouteGroup() if tc.expectError { require.Error(t, err) @@ -693,6 +754,21 @@ func TestStackGenerateRouteGroup(t *testing.T) { }, }, } + if tc.expectedBackend != nil { + expected.Spec.Backends = tc.expectedBackend + } + for _, be := range tc.expectedBackend { + found := false + for _, specBE := range rg.Spec.Backends { + if be.Type == specBE.Type { + found = true + break + } + } + if !found { + t.Fatalf("Failed to find backend %v, got %v", be, rg.Spec.Backends) + } + } require.Equal(t, expected, rg) }) } @@ -1236,7 +1312,7 @@ func TestStackGenerateDeployment(t *testing.T) { deploymentReplicas: 3, expectedReplicas: 1, stackAnnotations: map[string]string{ - forwardBackendAnnotation: "fwd1", + forwardBackendAnnotation: "fwd-deployment", }, }, } { @@ -1357,6 +1433,7 @@ func TestGenerateHPA(t *testing.T) { for _, tc := range []struct { name string autoscaler *zv1.Autoscaler + stackAnnotations map[string]string expectedMinReplicas *int32 expectedMaxReplicas int32 noTrafficSince time.Time @@ -1411,6 +1488,39 @@ func TestGenerateHPA(t *testing.T) { expectedMetrics: nil, expectedBehavior: nil, }, + { + name: "HPA when cluster migration should be scaled down", + autoscaler: &zv1.Autoscaler{ + MinReplicas: &min, + MaxReplicas: max, + + Metrics: []zv1.AutoscalerMetrics{ + { + Type: zv1.CPUAutoscalerMetric, + AverageUtilization: &utilization, + }, + }, + Behavior: exampleBehavior, + }, + stackAnnotations: map[string]string{ + forwardBackendAnnotation: "fwd-hpa", + }, + expectedMetrics: []autoscaling.MetricSpec{ + { + Type: autoscaling.ResourceMetricSourceType, + Resource: &autoscaling.ResourceMetricSource{ + Name: v1.ResourceCPU, + Target: autoscaling.MetricTarget{ + Type: autoscaling.UtilizationMetricType, + AverageUtilization: &utilization, + }, + }, + }, + }, + expectedBehavior: exampleBehavior, + expectedMinReplicas: &min, + expectedMaxReplicas: 1, + }, } { t.Run(tc.name, func(t *testing.T) { podTemplate := zv1.PodTemplateSpec{ @@ -1441,6 +1551,13 @@ func TestGenerateHPA(t *testing.T) { noTrafficSince: tc.noTrafficSince, scaledownTTL: time.Minute, } + if tc.stackAnnotations != nil { + if autoscalerContainer.Stack.Annotations != nil { + maps.Copy(autoscalerContainer.Stack.Annotations, tc.stackAnnotations) + } else { + autoscalerContainer.Stack.Annotations = tc.stackAnnotations + } + } hpa, err := autoscalerContainer.GenerateHPA() require.NoError(t, err) From dc42acc49da32870998a881ab7c7d032db5cf7c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 7 Nov 2025 19:31:59 +0100 Subject: [PATCH 13/17] fix: panic if stacktemplate annotations are nil MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- pkg/core/stackset.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/core/stackset.go b/pkg/core/stackset.go index 63eb8a59..a1b0cb17 100644 --- a/pkg/core/stackset.go +++ b/pkg/core/stackset.go @@ -83,7 +83,11 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) { spec.RouteGroup = ssc.StackSet.Spec.RouteGroup.DeepCopy() } - stackAnnotations := ssc.StackSet.Spec.StackTemplate.Annotations + stackAnnotations := make(map[string]string) + if a := ssc.StackSet.Spec.StackTemplate.Annotations; a != nil { + stackAnnotations = a + } + if forwardMigration { stackAnnotations[forwardBackendAnnotation] = forwardBackendName } From 1d7afafae185d5be324eb82e756ed9d0945e572b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 7 Nov 2025 19:50:38 +0100 Subject: [PATCH 14/17] fix: we do not need a deployment nor hpa if we migrate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- pkg/core/stack_resources.go | 19 ++++++++--------- pkg/core/stack_resources_test.go | 35 +++++++++++++++++--------------- pkg/core/stackset.go | 2 +- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/pkg/core/stack_resources.go b/pkg/core/stack_resources.go index 335481b3..6d923b23 100644 --- a/pkg/core/stack_resources.go +++ b/pkg/core/stack_resources.go @@ -191,6 +191,10 @@ func (sc *StackContainer) selector() map[string]string { // "zalando.org/forward-backend", the deployment will be set to // replicas 1. func (sc *StackContainer) GenerateDeployment() *appsv1.Deployment { + if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration { + return nil + } + stack := sc.Stack desiredReplicas := sc.stackReplicas @@ -246,11 +250,6 @@ func (sc *StackContainer) GenerateDeployment() *appsv1.Deployment { deployment.Spec.Strategy = *strategy } - if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration { - i := int32(1) - deployment.Spec.Replicas = &i - } - return deployment } @@ -262,6 +261,10 @@ func (sc *StackContainer) GenerateHPA() ( *autoscaling.HorizontalPodAutoscaler, error, ) { + if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration { + return nil, nil + } + autoscalerSpec := sc.Stack.Spec.StackSpec.Autoscaler trafficWeight := sc.actualTrafficWeight @@ -312,12 +315,6 @@ func (sc *StackContainer) GenerateHPA() ( result.Spec.MinReplicas = &pr } - if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration { - i := int32(1) - result.Spec.MinReplicas = &i - result.Spec.MaxReplicas = i - } - return result, nil } diff --git a/pkg/core/stack_resources_test.go b/pkg/core/stack_resources_test.go index b3714b1d..d56ee28a 100644 --- a/pkg/core/stack_resources_test.go +++ b/pkg/core/stack_resources_test.go @@ -1161,18 +1161,19 @@ func TestStackGenerateService(t *testing.T) { func TestStackGenerateDeployment(t *testing.T) { for _, tc := range []struct { - name string - hpaEnabled bool - stackReplicas int32 - minReadySeconds int32 - prescalingActive bool - prescalingReplicas int32 - deploymentReplicas int32 - noTrafficSince time.Time - expectedReplicas int32 - maxUnavailable int - maxSurge int - stackAnnotations map[string]string + name string + hpaEnabled bool + stackReplicas int32 + minReadySeconds int32 + prescalingActive bool + prescalingReplicas int32 + deploymentReplicas int32 + noTrafficSince time.Time + expectedReplicas int32 + maxUnavailable int + maxSurge int + stackAnnotations map[string]string + expectedDeploymentNil bool }{ { name: "stack scaled down to zero, deployment still running", @@ -1310,10 +1311,10 @@ func TestStackGenerateDeployment(t *testing.T) { name: "cluster migration should scale down deployment", stackReplicas: 3, deploymentReplicas: 3, - expectedReplicas: 1, stackAnnotations: map[string]string{ forwardBackendAnnotation: "fwd-deployment", }, + expectedDeploymentNil: true, }, } { t.Run(tc.name, func(t *testing.T) { @@ -1375,6 +1376,10 @@ func TestStackGenerateDeployment(t *testing.T) { } } deployment := c.GenerateDeployment() + if tc.expectedDeploymentNil { + require.Nil(t, deployment, "Failed to generate nil deployment") + return + } expected := &apps.Deployment{ ObjectMeta: testResourceMeta, Spec: apps.DeploymentSpec{ @@ -1517,9 +1522,7 @@ func TestGenerateHPA(t *testing.T) { }, }, }, - expectedBehavior: exampleBehavior, - expectedMinReplicas: &min, - expectedMaxReplicas: 1, + expectedBehavior: nil, }, } { t.Run(tc.name, func(t *testing.T) { diff --git a/pkg/core/stackset.go b/pkg/core/stackset.go index a1b0cb17..d96461a0 100644 --- a/pkg/core/stackset.go +++ b/pkg/core/stackset.go @@ -83,7 +83,7 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) { spec.RouteGroup = ssc.StackSet.Spec.RouteGroup.DeepCopy() } - stackAnnotations := make(map[string]string) + var stackAnnotations map[string]string if a := ssc.StackSet.Spec.StackTemplate.Annotations; a != nil { stackAnnotations = a } From dc95403fe3b416bb41bc0de5c2c30399030c4dc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 7 Nov 2025 20:01:27 +0100 Subject: [PATCH 15/17] fix panic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- pkg/core/stackset.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/core/stackset.go b/pkg/core/stackset.go index d96461a0..f985a630 100644 --- a/pkg/core/stackset.go +++ b/pkg/core/stackset.go @@ -83,7 +83,7 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) { spec.RouteGroup = ssc.StackSet.Spec.RouteGroup.DeepCopy() } - var stackAnnotations map[string]string + stackAnnotations := make(map[string]string) if a := ssc.StackSet.Spec.StackTemplate.Annotations; a != nil { stackAnnotations = a } @@ -91,6 +91,9 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) { if forwardMigration { stackAnnotations[forwardBackendAnnotation] = forwardBackendName } + if len(stackAnnotations) == 0 { + stackAnnotations = nil + } return &StackContainer{ Stack: &zv1.Stack{ From d1f7183d24533e17cd9b2340aee3bd9e1b2e51f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 7 Nov 2025 20:22:39 +0100 Subject: [PATCH 16/17] fix: override backends was using no ptr override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- pkg/core/stack_resources.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/core/stack_resources.go b/pkg/core/stack_resources.go index 6d923b23..e6258c4a 100644 --- a/pkg/core/stack_resources.go +++ b/pkg/core/stack_resources.go @@ -84,8 +84,8 @@ func patchForwardBackend(rg *rgv1.RouteGroupSpec) { Type: rgv1.ForwardRouteGroupBackend, }, } - for _, route := range rg.Routes { - route.Backends = []rgv1.RouteGroupBackendReference{ + for i := range rg.Routes { + rg.Routes[i].Backends = []rgv1.RouteGroupBackendReference{ { BackendName: forwardBackendName, }, From 0a93d0a7a6949e3bcc176d3d7aabeb8af24700e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 7 Nov 2025 20:52:57 +0100 Subject: [PATCH 17/17] fix routegroup test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- pkg/core/stack_resources.go | 5 ++ pkg/core/stack_resources_test.go | 80 ++++++++++++++++++++++++++------ 2 files changed, 71 insertions(+), 14 deletions(-) diff --git a/pkg/core/stack_resources.go b/pkg/core/stack_resources.go index e6258c4a..b42901f9 100644 --- a/pkg/core/stack_resources.go +++ b/pkg/core/stack_resources.go @@ -84,6 +84,11 @@ func patchForwardBackend(rg *rgv1.RouteGroupSpec) { Type: rgv1.ForwardRouteGroupBackend, }, } + rg.DefaultBackends = []rgv1.RouteGroupBackendReference{ + { + BackendName: forwardBackendName, + }, + } for i := range rg.Routes { rg.Routes[i].Backends = []rgv1.RouteGroupBackendReference{ { diff --git a/pkg/core/stack_resources_test.go b/pkg/core/stack_resources_test.go index d56ee28a..38b7841e 100644 --- a/pkg/core/stack_resources_test.go +++ b/pkg/core/stack_resources_test.go @@ -622,6 +622,7 @@ func TestStackGenerateRouteGroup(t *testing.T) { expectedAnnotations map[string]string expectedHosts []string expectedBackend []rgv1.RouteGroupBackend + expectedRouteGroup *rgv1.RouteGroup }{ { name: "no route group spec", @@ -691,6 +692,53 @@ func TestStackGenerateRouteGroup(t *testing.T) { Type: rgv1.ForwardRouteGroupBackend, }, }, + expectedRouteGroup: &rgv1.RouteGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-v1", + Namespace: "bar", + Annotations: map[string]string{ + stackGenerationAnnotationKey: "11", + "routegroup": "annotation", + }, + Labels: map[string]string{ + StacksetHeritageLabelKey: "foo", + StackVersionLabelKey: "v1", + "stack-label": "foobar", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: APIVersion, + Kind: KindStack, + Name: "foo-v1", + UID: "abc-123", + }, + }, + }, + Spec: rgv1.RouteGroupSpec{ + Hosts: []string{"foo-v1.example.org"}, + Backends: []rgv1.RouteGroupBackend{ + { + Name: "fwd", + Type: rgv1.ForwardRouteGroupBackend, + }, + }, + DefaultBackends: []rgv1.RouteGroupBackendReference{ + { + BackendName: "fwd", + }, + }, + Routes: []rgv1.RouteGroupRouteSpec{ + { + Backends: []rgv1.RouteGroupBackendReference{ + { + BackendName: "fwd", + }, + }, + PathSubtree: "/example", + }, + }, + }, + }, }, } { t.Run(tc.name, func(t *testing.T) { @@ -754,21 +802,25 @@ func TestStackGenerateRouteGroup(t *testing.T) { }, }, } - if tc.expectedBackend != nil { - expected.Spec.Backends = tc.expectedBackend - } - for _, be := range tc.expectedBackend { - found := false - for _, specBE := range rg.Spec.Backends { - if be.Type == specBE.Type { - found = true - break - } - } - if !found { - t.Fatalf("Failed to find backend %v, got %v", be, rg.Spec.Backends) - } + + if tc.expectedRouteGroup != nil { + expected = tc.expectedRouteGroup } + // if tc.expectedBackend != nil { + // expected.Spec.Backends = tc.expectedBackend + // } + // for _, be := range tc.expectedBackend { + // found := false + // for _, specBE := range rg.Spec.Backends { + // if be.Type == specBE.Type { + // found = true + // break + // } + // } + // if !found { + // t.Fatalf("Failed to find backend %v, got %v", be, rg.Spec.Backends) + // } + // } require.Equal(t, expected, rg) }) }