diff --git a/Makefile b/Makefile index 2e4ae7a070..8fd04efaf7 100644 --- a/Makefile +++ b/Makefile @@ -267,6 +267,18 @@ kitchensink-e2e: install-tools test-kitchensink-e2e: kitchensink-e2e +test-kitchensink-reinstall-testonly: + ./test/kitchensink-e2e-tests.sh --tags=reinstall -run TestServerlessReinstall + +test-kitchensink-reinstall: install-tools + UNINSTALL_STRIMZI="false" ./hack/strimzi.sh + SCALE_UP=5 INSTALL_KAFKA="true" ./hack/install.sh + ./test/kitchensink-e2e-tests.sh --tags=reinstall -run TestServerlessReinstallWithBrokerFeatures + ./test/kitchensink-e2e-tests.sh --tags=reinstall -run TestServerlessReinstallWithChannelFeatures + ./test/kitchensink-e2e-tests.sh --tags=reinstall -run TestServerlessReinstallWithSequenceFeatures + ./test/kitchensink-e2e-tests.sh --tags=reinstall -run TestServerlessReinstallWithSourceFeatures + ./test/kitchensink-e2e-tests.sh --tags=reinstall -run TestServerlessReinstallWithEventTransformFeatures + # Soak tests test-soak-testonly: ./test/soak-tests.sh diff --git a/test/kitchensinke2e/reinstall/reinstall.go b/test/kitchensinke2e/reinstall/reinstall.go new file mode 100644 index 0000000000..d8b1920c5b --- /dev/null +++ b/test/kitchensinke2e/reinstall/reinstall.go @@ -0,0 +1,209 @@ +package reinstall + +import ( + "context" + "testing" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + duckv1 "knative.dev/pkg/apis/duck/v1" + kubeclient "knative.dev/pkg/client/injection/kube/client" + "knative.dev/pkg/injection/clients/dynamicclient" + "knative.dev/reconciler-test/pkg/environment" + "knative.dev/reconciler-test/pkg/feature" + "knative.dev/reconciler-test/pkg/k8s" +) + +var ServingV1beta1 = schema.GroupVersionResource{Group: "operator.knative.dev", Version: "v1beta1", Resource: "knativeservings"} +var EventingV1beta1 = schema.GroupVersionResource{Group: "operator.knative.dev", Version: "v1beta1", Resource: "knativeeventings"} +var KnativeKafkaV1alpha1 = schema.GroupVersionResource{Group: "operator.serverless.openshift.io", Version: "v1alpha1", Resource: "knativekafkas"} + +var Serving = ServingV1beta1 +var Eventing = EventingV1beta1 +var KnativeKafka = KnativeKafkaV1alpha1 + +func uninstallResourceStep(gvr schema.GroupVersionResource, namespace string, name string) feature.StepFn { + return func(ctx context.Context, t feature.T) { + dynamicClient := dynamicclient.Get(ctx) + resourceStack := ResourceStackFromContext(ctx) + + serving, err := dynamicClient.Resource(gvr).Namespace(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + + resourceStack.Push(serving) + + err = dynamicClient.Resource(gvr).Namespace(namespace).Delete(ctx, name, metav1.DeleteOptions{}) + if err != nil { + t.Fatal(err) + } + } +} + +func waitForPodNonExistence(namespace string, podLabelsToNotExist ...string) feature.StepFn { + return func(ctx context.Context, t feature.T) { + kube := kubeclient.Get(ctx) + + interval, timeout := k8s.PollTimings(ctx, nil) + pods := kube.CoreV1().Pods(namespace) + + err := wait.PollImmediate(interval, timeout, func() (bool, error) { + for _, labelSelector := range podLabelsToNotExist { + plist, err := pods.List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) + if err != nil { + return false, err + } + if len(plist.Items) != 0 { + return false, nil + } + } + return true, nil + }) + + if err != nil { + t.Fatalf("Error waiting for pod non-existence in %s (%v): %v", namespace, podLabelsToNotExist, err) + } + } +} + +func waitForKnativeReadiness(namespace string, name string, gvr schema.GroupVersionResource) feature.StepFn { + return func(ctx context.Context, t feature.T) { + err := k8s.WaitForResourceCondition(ctx, t, namespace, name, gvr, func(resource duckv1.KResource) bool { + for _, condition := range resource.Status.Conditions { + if condition.Type == "Ready" && condition.Status == "True" { + return true + } + } + return false + }) + if err != nil { + t.Fatalf("Error waiting for Resource %s Readiness in %s (%v): %v", name, namespace, gvr, err) + } + } +} + +func uninstallKnativeServingStep() feature.StepFn { + return uninstallResourceStep(Serving, "knative-serving", "knative-serving") +} + +func uninstallKnativeEventingStep() feature.StepFn { + return uninstallResourceStep(Eventing, "knative-eventing", "knative-eventing") +} + +func uninstallKnativeKafkaStep() feature.StepFn { + return uninstallResourceStep(KnativeKafka, "knative-eventing", "knative-kafka") +} + +func reinstallResources() feature.StepFn { + return func(ctx context.Context, t feature.T) { + dynamicClient := dynamicclient.Get(ctx) + resourceStack := ResourceStackFromContext(ctx) + for resource := resourceStack.Pop(); resource != nil; resource = resourceStack.Pop() { + gvr, _ := meta.UnsafeGuessKindToResource(resource.GroupVersionKind()) + + // As we re-create the object, we clean up the Object metadata by creating a fresh one, copying the .spec + resourceSpec, _, _ := unstructured.NestedMap(resource.Object, "spec") + blankedUnstructured := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": resource.GroupVersionKind().GroupVersion().String(), + "kind": resource.GroupVersionKind().Kind, + "metadata": map[string]interface{}{ + "name": resource.GetName(), + "labels": resource.GetLabels(), + "annotations": resource.GetAnnotations(), + }, + "spec": resourceSpec, + }, + } + + _, err := dynamicClient.Resource(gvr).Namespace(resource.GetNamespace()).Create(ctx, blankedUnstructured, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + } + } +} + +func TestUninstalledFeatureSet(ctx context.Context, env environment.Environment, t *testing.T, fss ...feature.FeatureSet) { + t.Run("uninstall Serverless", func(t *testing.T) { + uninstallAll := feature.NewFeatureNamed("Uninstall Serverless") + // TODO: workaround for https://issues.redhat.com/browse/SRVKS-1325 , skip Serving reinstall for now + //uninstallAll.Setup("Uninstall Serving", uninstallKnativeServingStep()) + //uninstallAll.Assert("Wait for Serving controllers non-existence", waitForPodNonExistence("knative-serving", "app.kubernetes.io/component=controller", "app.kubernetes.io/component=webhook")) + + uninstallAll.Setup("Uninstall Eventing and KnativeKafka", func(ctx context.Context, t feature.T) { + // need to be sync, to be sure we install KnativeEventing first when re-installing + uninstallKnativeKafkaStep()(ctx, t) + uninstallKnativeEventingStep()(ctx, t) + }) + uninstallAll.Assert("Wait for Eventing controllers non-existence", waitForPodNonExistence("knative-eventing", "app.kubernetes.io/component=eventing-controller", "app.kubernetes.io/component=eventing-webhook")) + uninstallAll.Assert("Wait for Eventing Kafka controllers non-existence", waitForPodNonExistence("knative-eventing", "app.kubernetes.io/component=kafka-controller", "app.kubernetes.io/component=kafka-webhook-eventing")) + + env.Test(ctx, t, uninstallAll) + }) + + t.Run("setups", func(t *testing.T) { + for _, fs := range fss { + fs := fs + for _, f := range fs.Features { + f := f + + preInstallSteps := make([]feature.Step, 0, len(f.Steps)) + + for _, s := range f.Steps { + if s.T != feature.Assert && s.T != feature.Teardown { + preInstallSteps = append(preInstallSteps, s) + } + } + + preInstallFeature := feature.NewFeatureNamed(f.Name + " pre-install steps") + preInstallFeature.Steps = preInstallSteps + + // Run features within feature sets in parallel. + t.Run(f.Name, func(t *testing.T) { + t.Parallel() + env.Test(ctx, t, preInstallFeature) + }) + } + } + }) + + t.Run("reinstall Serverless", func(t *testing.T) { + reinstallServerless := feature.NewFeatureNamed("Reinstall Serverless") + reinstallServerless.Setup("Reinstall Serverless", reinstallResources()) + reinstallServerless.Assert("Wait for KnativeServing Readiness", waitForKnativeReadiness("knative-serving", "knative-serving", Serving)) + reinstallServerless.Assert("Wait for KnativeEventing Readiness", waitForKnativeReadiness("knative-eventing", "knative-eventing", Eventing)) + reinstallServerless.Assert("Wait for KnativeKafka Readiness", waitForKnativeReadiness("knative-eventing", "knative-kafka", KnativeKafka)) + env.Test(ctx, t, reinstallServerless) + }) + + t.Run("assertions", func(t *testing.T) { + for _, fs := range fss { + fs := fs + for _, f := range fs.Features { + f := f + + postInstallSteps := make([]feature.Step, 0, len(f.Steps)) + + for _, s := range f.Steps { + if s.T == feature.Assert || s.T == feature.Teardown { + postInstallSteps = append(postInstallSteps, s) + } + } + + postInstallFeature := feature.NewFeatureNamed(f.Name + " post-install steps") + postInstallFeature.Steps = postInstallSteps + + // Run features within feature sets in parallel. + t.Run(f.Name, func(t *testing.T) { + t.Parallel() + env.Test(ctx, t, postInstallFeature) + }) + } + } + }) +} diff --git a/test/kitchensinke2e/reinstall/resourcestack.go b/test/kitchensinke2e/reinstall/resourcestack.go new file mode 100644 index 0000000000..025bcedcd8 --- /dev/null +++ b/test/kitchensinke2e/reinstall/resourcestack.go @@ -0,0 +1,48 @@ +package reinstall + +import ( + "context" + "sync" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +/* +ResourceStack stores a stack of resources in Context. +The original use case is the re-install test, in which part of the installed resources are removed from a cluster +and then re-installed again later in the test. +*/ +type ResourceStack struct { + stack []*unstructured.Unstructured + mu sync.Mutex +} + +func (f *ResourceStack) Push(unstructured *unstructured.Unstructured) { + f.mu.Lock() + defer f.mu.Unlock() + f.stack = append(f.stack, unstructured) +} + +func (f *ResourceStack) Pop() *unstructured.Unstructured { + f.mu.Lock() + defer f.mu.Unlock() + if len(f.stack) == 0 { + return nil + } + ret := f.stack[len(f.stack)-1] + f.stack = f.stack[:len(f.stack)-1] + return ret +} + +type resourceStackKey struct{} + +func ContextWithResourceStack(ctx context.Context, store *ResourceStack) context.Context { + return context.WithValue(ctx, resourceStackKey{}, store) +} + +func ResourceStackFromContext(ctx context.Context) *ResourceStack { + if e, ok := ctx.Value(resourceStackKey{}).(*ResourceStack); ok { + return e + } + panic("no ResourceStack found in context") +} diff --git a/test/kitchensinke2e/reinstall_test.go b/test/kitchensinke2e/reinstall_test.go new file mode 100644 index 0000000000..616e1f973f --- /dev/null +++ b/test/kitchensinke2e/reinstall_test.go @@ -0,0 +1,72 @@ +//go:build reinstall +// +build reinstall + +package kitchensinke2e + +import ( + "context" + "testing" + "time" + + "github.com/openshift-knative/serverless-operator/test/kitchensinke2e/features" + "github.com/openshift-knative/serverless-operator/test/kitchensinke2e/reinstall" + "knative.dev/pkg/system" + "knative.dev/reconciler-test/pkg/environment" + "knative.dev/reconciler-test/pkg/feature" + "knative.dev/reconciler-test/pkg/k8s" + "knative.dev/reconciler-test/pkg/knative" +) + +// Using custom env for re-install tests because of the need of the "resource stack" in the context +func testUninstalledFeatureSet(t *testing.T, fss ...feature.FeatureSet) { + ctx, env := global.Environment( + knative.WithKnativeNamespace(system.Namespace()), + knative.WithLoggingConfig, + knative.WithTracingConfig, + k8s.WithEventListener, + environment.WithPollTimings(4*time.Second, 10*time.Minute), + environment.Managed(t), + func(ctx context.Context, env environment.Environment) (context.Context, error) { + return reinstall.ContextWithResourceStack(ctx, new(reinstall.ResourceStack)), nil + }, + ) + + reinstall.TestUninstalledFeatureSet(ctx, env, t, fss...) +} + +func TestServerlessReinstallWithBrokerFeatures(t *testing.T) { + t.Skip("https://issues.redhat.com/browse/SRVKE-1808 known issues with Broker and reinstall") + + // Split the big Broker featuresets + for _, fs := range split(features.BrokerFeatureSetWithBrokerDLS(), groupSize) { + t.Run(fs.Name, func(t *testing.T) { + testUninstalledFeatureSet(t, fs) + }) + } + + for _, fs := range split(features.BrokerFeatureSetWithTriggerDLS(), groupSize) { + t.Run(fs.Name, func(t *testing.T) { + testUninstalledFeatureSet(t, fs) + }) + } +} + +func TestServerlessReinstallWithChannelFeatures(t *testing.T) { + testUninstalledFeatureSet(t, features.ChannelFeatureSet()) +} + +func TestServerlessReinstallWithSequenceFeatures(t *testing.T) { + // join the small sequence feature sets + testUninstalledFeatureSet(t, features.SequenceNoReplyFeatureSet(), + features.ParallelNoReplyFeatureSet(), + features.SequenceGlobalReplyFeatureSet(), + features.ParallelGlobalReplyFeatureSet()) +} + +func TestServerlessReinstallWithSourceFeatures(t *testing.T) { + testUninstalledFeatureSet(t, features.SourceFeatureSet()) +} + +func TestServerlessReinstallWithEventTransformFeatures(t *testing.T) { + testUninstalledFeatureSet(t, features.EventTransformFeatureSet()) +}