diff --git a/docs/stack_crd.yaml b/docs/stack_crd.yaml index deece1e6..801f6d5b 100644 --- a/docs/stack_crd.yaml +++ b/docs/stack_crd.yaml @@ -475,6 +475,22 @@ spec: - type: integer - type: string x-kubernetes-int-or-string: true + metadata: + description: |- + EmbeddedObjectMetaWithAnnotations defines the metadata which can be attached + to a resource. It's a slimmed down version of metav1.ObjectMeta only + containing annotations. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + type: object required: - backendPort type: object diff --git a/docs/stackset_crd.yaml b/docs/stackset_crd.yaml index 3213f79d..ea242252 100644 --- a/docs/stackset_crd.yaml +++ b/docs/stackset_crd.yaml @@ -67,6 +67,22 @@ spec: - type: integer - type: string x-kubernetes-int-or-string: true + metadata: + description: |- + EmbeddedObjectMetaWithAnnotations defines the metadata which can be attached + to a resource. It's a slimmed down version of metav1.ObjectMeta only + containing annotations. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + type: object required: - backendPort type: object @@ -3589,12 +3605,6 @@ spec: Must be set if and only if type is "Localhost". type: string type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. type: string required: - type diff --git a/pkg/apis/zalando.org/v1/types.go b/pkg/apis/zalando.org/v1/types.go index 74191701..8492ea3e 100644 --- a/pkg/apis/zalando.org/v1/types.go +++ b/pkg/apis/zalando.org/v1/types.go @@ -124,7 +124,8 @@ func (s *StackSetIngressSpec) GetAnnotations() map[string]string { // backendport for ingress managed outside of stackset. // +k8s:deepcopy-gen=true type StackSetExternalIngressSpec struct { - BackendPort intstr.IntOrString `json:"backendPort"` + EmbeddedObjectMetaWithAnnotations `json:"metadata,omitempty"` + BackendPort intstr.IntOrString `json:"backendPort"` } // RouteGroupSpec defines the specification for defining a RouteGroup attached diff --git a/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go b/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go index c55d3c03..fbc4a70c 100644 --- a/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go +++ b/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go @@ -745,6 +745,7 @@ func (in *StackSet) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StackSetExternalIngressSpec) DeepCopyInto(out *StackSetExternalIngressSpec) { *out = *in + in.EmbeddedObjectMetaWithAnnotations.DeepCopyInto(&out.EmbeddedObjectMetaWithAnnotations) out.BackendPort = in.BackendPort return } @@ -826,7 +827,7 @@ func (in *StackSetSpec) DeepCopyInto(out *StackSetSpec) { if in.ExternalIngress != nil { in, out := &in.ExternalIngress, &out.ExternalIngress *out = new(StackSetExternalIngressSpec) - **out = **in + (*in).DeepCopyInto(*out) } if in.RouteGroup != nil { in, out := &in.RouteGroup, &out.RouteGroup @@ -937,7 +938,7 @@ func (in *StackSpecInternal) DeepCopyInto(out *StackSpecInternal) { if in.ExternalIngress != nil { in, out := &in.ExternalIngress, &out.ExternalIngress *out = new(StackSetExternalIngressSpec) - **out = **in + (*in).DeepCopyInto(*out) } if in.Ingress != nil { in, out := &in.Ingress, &out.Ingress diff --git a/pkg/core/stack_resources.go b/pkg/core/stack_resources.go index aee61911..b42901f9 100644 --- a/pkg/core/stack_resources.go +++ b/pkg/core/stack_resources.go @@ -76,6 +76,28 @@ func objectMetaInjectLabels(objectMeta metav1.ObjectMeta, labels map[string]stri return objectMeta } +// patchForwardBackend rewrites a RouteGroupSpec to send all traffic to another cluster chosen by the operator of skipper +func patchForwardBackend(rg *rgv1.RouteGroupSpec) { + rg.Backends = []rgv1.RouteGroupBackend{ + { + Name: forwardBackendName, + Type: rgv1.ForwardRouteGroupBackend, + }, + } + rg.DefaultBackends = []rgv1.RouteGroupBackendReference{ + { + BackendName: forwardBackendName, + }, + } + for i := range rg.Routes { + rg.Routes[i].Backends = []rgv1.RouteGroupBackendReference{ + { + BackendName: forwardBackendName, + }, + } + } +} + func (sc *StackContainer) objectMeta(segment bool) metav1.ObjectMeta { resourceLabels := mapCopy(sc.Stack.Labels) @@ -169,7 +191,15 @@ func (sc *StackContainer) selector() map[string]string { return limitLabels(sc.Stack.Labels, selectorLabels) } +// GenerateDeployment generates a deployment as configured in the +// stack. On cluster migrations set by stackset annotation +// "zalando.org/forward-backend", the deployment will be set to +// replicas 1. func (sc *StackContainer) GenerateDeployment() *appsv1.Deployment { + if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration { + return nil + } + stack := sc.Stack desiredReplicas := sc.stackReplicas @@ -224,13 +254,22 @@ func (sc *StackContainer) GenerateDeployment() *appsv1.Deployment { if strategy != nil { deployment.Spec.Strategy = *strategy } + return deployment } +// GenerateHPA generates a hpa as configured in the +// stack. On cluster migrations set by stackset annotation +// "zalando.org/forward-backend", the hpa will be set to +// minReplicas = maxReplicass = 1. func (sc *StackContainer) GenerateHPA() ( *autoscaling.HorizontalPodAutoscaler, error, ) { + if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration { + return nil, nil + } + autoscalerSpec := sc.Stack.Spec.StackSpec.Autoscaler trafficWeight := sc.actualTrafficWeight @@ -379,6 +418,10 @@ func (sc *StackContainer) GenerateIngressSegment() ( return res, nil } +// generateIngress generates an ingress as configured in the stack. +// On cluster migrations set by stackset annotation +// "zalando.org/forward-backend", the annotation will be copied to the +// ingress. func (sc *StackContainer) generateIngress(segment bool) ( *networking.Ingress, error, @@ -430,6 +473,10 @@ func (sc *StackContainer) generateIngress(segment bool) ( Rules: rules, }, } + if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration { + // see https://opensource.zalando.com/skipper/kubernetes/ingress-usage/#skipper-ingress-annotations + result.Annotations["zalando.org/skipper-backend"] = "forward" + } // insert annotations result.Annotations = mergeLabels( @@ -470,6 +517,10 @@ func (sc *StackContainer) GenerateRouteGroupSegment() ( return res, nil } +// generateRouteGroup generates an RouteGroup as configured in the +// stack. On cluster migrations set by stackset annotation +// "zalando.org/forward-backend", the RouteGroup will be patched by +// patchForwardBackend() to execute the migration. func (sc *StackContainer) generateRouteGroup(segment bool) ( *rgv1.RouteGroup, error, @@ -529,6 +580,10 @@ func (sc *StackContainer) generateRouteGroup(segment bool) ( sc.routeGroupSpec.GetAnnotations(), ) + if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration { + patchForwardBackend(&result.Spec) + } + return result, nil } diff --git a/pkg/core/stack_resources_test.go b/pkg/core/stack_resources_test.go index 80d2bbba..38b7841e 100644 --- a/pkg/core/stack_resources_test.go +++ b/pkg/core/stack_resources_test.go @@ -2,6 +2,7 @@ package core import ( "errors" + "maps" "reflect" "slices" "strconv" @@ -281,8 +282,9 @@ func TestLimitLabels(t *testing.T) { func TestStackGenerateIngress(t *testing.T) { for _, tc := range []struct { - name string - ingressSpec *zv1.StackSetIngressSpec + name string + ingressSpec *zv1.StackSetIngressSpec + stackAnnotations map[string]string expectDisabled bool expectError bool @@ -309,6 +311,25 @@ func TestStackGenerateIngress(t *testing.T) { }, expectedHosts: []string{"foo-v1.example.org"}, }, + { + name: "cluster migration", + ingressSpec: &zv1.StackSetIngressSpec{ + EmbeddedObjectMetaWithAnnotations: zv1.EmbeddedObjectMetaWithAnnotations{ + Annotations: map[string]string{"ingress": "annotation"}, + }, + Hosts: []string{"foo.example.org", "foo.example.com"}, + Path: "example", + }, + stackAnnotations: map[string]string{ + forwardBackendAnnotation: "fwd-ingress", + }, + expectedAnnotations: map[string]string{ + stackGenerationAnnotationKey: "11", + "ingress": "annotation", + "zalando.org/skipper-backend": "forward", + }, + expectedHosts: []string{"foo-v1.example.org"}, + }, } { t.Run(tc.name, func(t *testing.T) { backendPort := int32(80) @@ -322,6 +343,10 @@ func TestStackGenerateIngress(t *testing.T) { backendPort: &intStrBackendPort, clusterDomains: []string{"example.org"}, } + if tc.stackAnnotations != nil { + c.Stack.Annotations = tc.stackAnnotations + } + ingress, err := c.GenerateIngress() if tc.expectError { @@ -591,10 +616,13 @@ func TestStackGenerateRouteGroup(t *testing.T) { for _, tc := range []struct { name string routeGroupSpec *zv1.RouteGroupSpec + stackAnnotations map[string]string expectDisabled bool expectError bool expectedAnnotations map[string]string expectedHosts []string + expectedBackend []rgv1.RouteGroupBackend + expectedRouteGroup *rgv1.RouteGroup }{ { name: "no route group spec", @@ -637,6 +665,81 @@ func TestStackGenerateRouteGroup(t *testing.T) { }, expectedHosts: []string{"foo-v1.example.org"}, }, + { + name: "cluster migration", + routeGroupSpec: &zv1.RouteGroupSpec{ + EmbeddedObjectMetaWithAnnotations: zv1.EmbeddedObjectMetaWithAnnotations{ + Annotations: map[string]string{"routegroup": "annotation"}, + }, + Hosts: []string{"foo.example.org", "foo.example.com"}, + Routes: []rgv1.RouteGroupRouteSpec{ + { + PathSubtree: "/example", + }, + }, + }, + stackAnnotations: map[string]string{ + forwardBackendAnnotation: "fwd-routegroup", + }, + expectedAnnotations: map[string]string{ + stackGenerationAnnotationKey: "11", + "routegroup": "annotation", + }, + expectedHosts: []string{"foo-v1.example.org"}, + expectedBackend: []rgv1.RouteGroupBackend{ + { + Name: "fwd", + Type: rgv1.ForwardRouteGroupBackend, + }, + }, + expectedRouteGroup: &rgv1.RouteGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-v1", + Namespace: "bar", + Annotations: map[string]string{ + stackGenerationAnnotationKey: "11", + "routegroup": "annotation", + }, + Labels: map[string]string{ + StacksetHeritageLabelKey: "foo", + StackVersionLabelKey: "v1", + "stack-label": "foobar", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: APIVersion, + Kind: KindStack, + Name: "foo-v1", + UID: "abc-123", + }, + }, + }, + Spec: rgv1.RouteGroupSpec{ + Hosts: []string{"foo-v1.example.org"}, + Backends: []rgv1.RouteGroupBackend{ + { + Name: "fwd", + Type: rgv1.ForwardRouteGroupBackend, + }, + }, + DefaultBackends: []rgv1.RouteGroupBackendReference{ + { + BackendName: "fwd", + }, + }, + Routes: []rgv1.RouteGroupRouteSpec{ + { + Backends: []rgv1.RouteGroupBackendReference{ + { + BackendName: "fwd", + }, + }, + PathSubtree: "/example", + }, + }, + }, + }, + }, } { t.Run(tc.name, func(t *testing.T) { backendPort := int32(80) @@ -650,6 +753,13 @@ func TestStackGenerateRouteGroup(t *testing.T) { backendPort: &intStrBackendPort, clusterDomains: []string{"example.org"}, } + if tc.stackAnnotations != nil { + if c.Stack.Annotations != nil { + maps.Copy(c.Stack.Annotations, tc.stackAnnotations) + } else { + c.Stack.Annotations = tc.stackAnnotations + } + } rg, err := c.GenerateRouteGroup() if tc.expectError { require.Error(t, err) @@ -692,6 +802,25 @@ func TestStackGenerateRouteGroup(t *testing.T) { }, }, } + + if tc.expectedRouteGroup != nil { + expected = tc.expectedRouteGroup + } + // if tc.expectedBackend != nil { + // expected.Spec.Backends = tc.expectedBackend + // } + // for _, be := range tc.expectedBackend { + // found := false + // for _, specBE := range rg.Spec.Backends { + // if be.Type == specBE.Type { + // found = true + // break + // } + // } + // if !found { + // t.Fatalf("Failed to find backend %v, got %v", be, rg.Spec.Backends) + // } + // } require.Equal(t, expected, rg) }) } @@ -830,15 +959,7 @@ func TestStackGenerateRouteGroupSegment(t *testing.T) { } for _, r := range rg.Spec.Routes { - found := false - for _, p := range r.Predicates { - if p == tc.expectedPredicate { - found = true - break - } - } - - if !found { + if !slices.Contains(r.Predicates, tc.expectedPredicate) { t.Errorf("predicate %q not found in route %v", tc.expectedPredicate, r, @@ -1092,17 +1213,19 @@ func TestStackGenerateService(t *testing.T) { func TestStackGenerateDeployment(t *testing.T) { for _, tc := range []struct { - name string - hpaEnabled bool - stackReplicas int32 - minReadySeconds int32 - prescalingActive bool - prescalingReplicas int32 - deploymentReplicas int32 - noTrafficSince time.Time - expectedReplicas int32 - maxUnavailable int - maxSurge int + name string + hpaEnabled bool + stackReplicas int32 + minReadySeconds int32 + prescalingActive bool + prescalingReplicas int32 + deploymentReplicas int32 + noTrafficSince time.Time + expectedReplicas int32 + maxUnavailable int + maxSurge int + stackAnnotations map[string]string + expectedDeploymentNil bool }{ { name: "stack scaled down to zero, deployment still running", @@ -1236,6 +1359,15 @@ func TestStackGenerateDeployment(t *testing.T) { name: "minReadySeconds should be set", minReadySeconds: 5, }, + { + name: "cluster migration should scale down deployment", + stackReplicas: 3, + deploymentReplicas: 3, + stackAnnotations: map[string]string{ + forwardBackendAnnotation: "fwd-deployment", + }, + expectedDeploymentNil: true, + }, } { t.Run(tc.name, func(t *testing.T) { var strategy *apps.DeploymentStrategy @@ -1288,7 +1420,18 @@ func TestStackGenerateDeployment(t *testing.T) { if tc.hpaEnabled { c.Stack.Spec.StackSpec.Autoscaler = &zv1.Autoscaler{} } + if tc.stackAnnotations != nil { + if c.Stack.Annotations != nil { + maps.Copy(c.Stack.Annotations, tc.stackAnnotations) + } else { + c.Stack.Annotations = tc.stackAnnotations + } + } deployment := c.GenerateDeployment() + if tc.expectedDeploymentNil { + require.Nil(t, deployment, "Failed to generate nil deployment") + return + } expected := &apps.Deployment{ ObjectMeta: testResourceMeta, Spec: apps.DeploymentSpec{ @@ -1347,6 +1490,7 @@ func TestGenerateHPA(t *testing.T) { for _, tc := range []struct { name string autoscaler *zv1.Autoscaler + stackAnnotations map[string]string expectedMinReplicas *int32 expectedMaxReplicas int32 noTrafficSince time.Time @@ -1401,6 +1545,37 @@ func TestGenerateHPA(t *testing.T) { expectedMetrics: nil, expectedBehavior: nil, }, + { + name: "HPA when cluster migration should be scaled down", + autoscaler: &zv1.Autoscaler{ + MinReplicas: &min, + MaxReplicas: max, + + Metrics: []zv1.AutoscalerMetrics{ + { + Type: zv1.CPUAutoscalerMetric, + AverageUtilization: &utilization, + }, + }, + Behavior: exampleBehavior, + }, + stackAnnotations: map[string]string{ + forwardBackendAnnotation: "fwd-hpa", + }, + expectedMetrics: []autoscaling.MetricSpec{ + { + Type: autoscaling.ResourceMetricSourceType, + Resource: &autoscaling.ResourceMetricSource{ + Name: v1.ResourceCPU, + Target: autoscaling.MetricTarget{ + Type: autoscaling.UtilizationMetricType, + AverageUtilization: &utilization, + }, + }, + }, + }, + expectedBehavior: nil, + }, } { t.Run(tc.name, func(t *testing.T) { podTemplate := zv1.PodTemplateSpec{ @@ -1431,6 +1606,13 @@ func TestGenerateHPA(t *testing.T) { noTrafficSince: tc.noTrafficSince, scaledownTTL: time.Minute, } + if tc.stackAnnotations != nil { + if autoscalerContainer.Stack.Annotations != nil { + maps.Copy(autoscalerContainer.Stack.Annotations, tc.stackAnnotations) + } else { + autoscalerContainer.Stack.Annotations = tc.stackAnnotations + } + } hpa, err := autoscalerContainer.GenerateHPA() require.NoError(t, err) diff --git a/pkg/core/stackset.go b/pkg/core/stackset.go index 1fe5867e..f985a630 100644 --- a/pkg/core/stackset.go +++ b/pkg/core/stackset.go @@ -17,6 +17,8 @@ const ( StackVersionLabelKey = "stack-version" ingressTrafficAuthoritativeAnnotation = "zalando.org/traffic-authoritative" + forwardBackendAnnotation = "zalando.org/forward-backend" + forwardBackendName = "fwd" ) var ( @@ -52,6 +54,7 @@ func sanitizeServicePorts(service *zv1.StackServiceSpec) *zv1.StackServiceSpec { // NewStack returns an (optional) stack that should be created func (ssc *StackSetContainer) NewStack() (*StackContainer, string) { + _, forwardMigration := ssc.StackSet.ObjectMeta.Annotations[forwardBackendAnnotation] observedStackVersion := ssc.StackSet.Status.ObservedStackVersion stackVersion := currentStackVersion(ssc.StackSet) stackName := generateStackName(ssc.StackSet, stackVersion) @@ -80,6 +83,18 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) { spec.RouteGroup = ssc.StackSet.Spec.RouteGroup.DeepCopy() } + stackAnnotations := make(map[string]string) + if a := ssc.StackSet.Spec.StackTemplate.Annotations; a != nil { + stackAnnotations = a + } + + if forwardMigration { + stackAnnotations[forwardBackendAnnotation] = forwardBackendName + } + if len(stackAnnotations) == 0 { + stackAnnotations = nil + } + return &StackContainer{ Stack: &zv1.Stack{ ObjectMeta: metav1.ObjectMeta{ @@ -100,7 +115,7 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) { ssc.StackSet.Labels, map[string]string{StackVersionLabelKey: stackVersion}, ), - Annotations: ssc.StackSet.Spec.StackTemplate.Annotations, + Annotations: stackAnnotations, }, Spec: *spec, },