diff --git a/test/e2e/baseline_test.go b/test/e2e/dual-engine/baseline_test.go similarity index 100% rename from test/e2e/baseline_test.go rename to test/e2e/dual-engine/baseline_test.go diff --git a/test/e2e/gateway_test.go b/test/e2e/dual-engine/gateway_test.go similarity index 100% rename from test/e2e/gateway_test.go rename to test/e2e/dual-engine/gateway_test.go diff --git a/test/e2e/main_test.go b/test/e2e/dual-engine/main_test.go similarity index 100% rename from test/e2e/main_test.go rename to test/e2e/dual-engine/main_test.go diff --git a/test/e2e/manage_test.go b/test/e2e/dual-engine/manage_test.go similarity index 100% rename from test/e2e/manage_test.go rename to test/e2e/dual-engine/manage_test.go diff --git a/test/e2e/restart_test.go b/test/e2e/dual-engine/restart_test.go similarity index 100% rename from test/e2e/restart_test.go rename to test/e2e/dual-engine/restart_test.go diff --git a/test/e2e/kernel-native/baseline_test.go b/test/e2e/kernel-native/baseline_test.go new file mode 100644 index 000000000..880f82377 --- /dev/null +++ b/test/e2e/kernel-native/baseline_test.go @@ -0,0 +1,84 @@ +//go:build integ || all +// +build integ all + +/* + * Copyright The Kmesh Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// NOTE: THE CODE IN THIS FILE IS MAINLY REFERENCED FROM ISTIO INTEGRATION +// FRAMEWORK(https://github.com/istio/istio/tree/master/tests/integration) +// AND ADAPTED FOR KMESH. + +package kmesh + +import ( + "fmt" + "net/http" + + echot "istio.io/istio/pkg/test/echo" + "istio.io/istio/pkg/test/echo/common/scheme" + "istio.io/istio/pkg/test/framework" + "istio.io/istio/pkg/test/framework/components/echo" + "istio.io/istio/pkg/test/framework/components/echo/check" + "istio.io/istio/pkg/util/sets" +) + +func IsL7() echo.Checker { + return nil +} + +func IsL4() echo.Checker { + return check.Each(func(r echot.Response) error { + // TODO: response headers? + _, f := r.RequestHeaders[http.CanonicalHeaderKey("X-Request-Id")] + if f { + return fmt.Errorf("X-Request-Id set, is L7 processing enabled unexpectedly?") + } + return nil + }) +} + +var ( + httpValidator = check.And(check.OK(), IsL7()) + tcpValidator = check.And(check.OK(), IsL4()) + callOptions = []echo.CallOptions{ + { + Port: echo.Port{Name: "http"}, + Scheme: scheme.HTTP, + Count: 10, + }, + { + Port: echo.Port{Name: "tcp"}, + Scheme: scheme.TCP, + Count: 1, + }, + } +) + +func supportsL7(opt echo.CallOptions, src, dst echo.Instance) bool { + isL7Scheme := opt.Scheme == scheme.HTTP || opt.Scheme == scheme.GRPC || opt.Scheme == scheme.WebSocket + return dst.Config().HasAnyWaypointProxy() && isL7Scheme +} + +func OriginalSourceCheck(t framework.TestContext, src echo.Instance) echo.Checker { + // Check that each response saw one of the workload IPs for the src echo instance + addresses := sets.New(src.WorkloadsOrFail(t).Addresses()...) + return check.Each(func(response echot.Response) error { + if !addresses.Contains(response.IP) { + return fmt.Errorf("expected original source (%v) to be propagated, but got %v", addresses.UnsortedList(), response.IP) + } + return nil + }) +} diff --git a/test/e2e/kernel-native/main_test.go b/test/e2e/kernel-native/main_test.go new file mode 100644 index 000000000..16eed9a98 --- /dev/null +++ b/test/e2e/kernel-native/main_test.go @@ -0,0 +1,133 @@ +//go:build integ || all +// +build integ all + +/* + * Copyright The Kmesh Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kmesh + +import ( + "testing" + "time" + + "istio.io/api/label" + "istio.io/istio/pkg/test/framework" + "istio.io/istio/pkg/test/framework/components/echo" + "istio.io/istio/pkg/test/framework/components/echo/common/ports" + "istio.io/istio/pkg/test/framework/components/echo/deployment" + "istio.io/istio/pkg/test/framework/components/echo/match" + "istio.io/istio/pkg/test/framework/components/namespace" + "istio.io/istio/pkg/test/framework/resource" +) + +var ( + apps = &EchoDeployments{} +) + +type EchoDeployments struct { + Namespace namespace.Instance + All echo.Instances + + // The echo service which is enrolled to Kmesh without waypoint. + EnrolledToKmesh echo.Instances +} + +const ( + ServiceName = "echo-service" + ServiceName2 = "echo-service2" + EnrolledToKmesh = "enrolled-to-kmesh" + Timeout = 2 * time.Minute + KmeshReleaseName = "kmesh" + KmeshDaemonsetName = "kmesh" + KmeshNamespace = "kmesh-system" + DataplaneModeKmesh = "Kmesh" +) + +func TestMain(m *testing.M) { + framework. + NewSuite(m). + Setup(func(t resource.Context) error { + t.Settings().Ambient = true + return nil + }). + Setup(func(t resource.Context) error { + return SetupApps(t, apps) + }). + Run() +} + +func SetupApps(t resource.Context, apps *EchoDeployments) error { + var err error + apps.Namespace, err = namespace.New(t, namespace.Config{ + Prefix: "echo", + Inject: false, + Labels: map[string]string{ + label.IoIstioDataplaneMode.Name: DataplaneModeKmesh, + }, + }) + if err != nil { + return err + } + + builder := deployment.New(t). + WithClusters(t.Clusters()...). + WithConfig(echo.Config{ + Service: ServiceName, + Namespace: apps.Namespace, + Ports: ports.All(), + ServiceAccount: true, + Subsets: []echo.SubsetConfig{ + { + Replicas: 1, + Version: "v1", + Labels: map[string]string{ + "app": ServiceName, + "version": "v1", + }, + }, + { + Replicas: 1, + Version: "v2", + Labels: map[string]string{ + "app": ServiceName, + "version": "v2", + }, + }, + }, + }). + WithConfig(echo.Config{ + Service: EnrolledToKmesh, + Namespace: apps.Namespace, + Ports: ports.All(), + ServiceAccount: true, + Subsets: []echo.SubsetConfig{ + { + Replicas: 1, + Version: "v1", + }, + }, + }) + + echos, err := builder.Build() + if err != nil { + return err + } + + apps.All = echos + + apps.EnrolledToKmesh = match.ServiceName(echo.NamespacedName{Name: EnrolledToKmesh, Namespace: apps.Namespace}).GetMatches(echos) + return nil +} diff --git a/test/e2e/kernel-native/restart_test.go b/test/e2e/kernel-native/restart_test.go new file mode 100644 index 000000000..c8f1191a3 --- /dev/null +++ b/test/e2e/kernel-native/restart_test.go @@ -0,0 +1,107 @@ +//go:build integ || all +// +build integ all + +/* + * Copyright The Kmesh Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// NOTE: THE CODE IN THIS FILE IS MAINLY REFERENCED FROM ISTIO INTEGRATION +// FRAMEWORK(https://github.com/istio/istio/tree/master/tests/integration) +// AND ADAPTED FOR KMESH. + +package kmesh + +import ( + "context" + "fmt" + "testing" + "time" + + "istio.io/istio/pkg/test/framework" + "istio.io/istio/pkg/test/framework/components/echo" + "istio.io/istio/pkg/test/framework/components/echo/util/traffic" + kubetest "istio.io/istio/pkg/test/kube" + "istio.io/istio/pkg/test/util/retry" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +func TestKmeshRestart(t *testing.T) { + framework.NewTest(t).Run(func(t framework.TestContext) { + src := apps.EnrolledToKmesh[0] + dst := apps.All + options := echo.CallOptions{ + To: dst, + Count: 1, + // Determine whether it is managed by Kmesh by passing through Waypoint. + Check: httpValidator, + Port: echo.Port{ + Name: "http", + }, + Retry: echo.Retry{NoRetry: true}, + } + + g := traffic.NewGenerator(t, traffic.Config{ + Source: src, + Options: options, + Interval: 50 * time.Millisecond, + }).Start() + + time.Sleep(5 * time.Second) + restartKmesh(t) + g.Stop().CheckSuccessRate(t, 0.1) + }) +} + +func restartKmesh(t framework.TestContext) { + patchOpts := metav1.PatchOptions{} + patchData := fmt.Sprintf(`{ + "spec": { + "template": { + "metadata": { + "annotations": { + "kubectl.kubernetes.io/restartedAt": %q + } + } + } + } + }`, time.Now().Format(time.RFC3339)) + ds := t.Clusters().Default().Kube().AppsV1().DaemonSets(KmeshNamespace) + _, err := ds.Patch(context.Background(), KmeshDaemonsetName, types.StrategicMergePatchType, []byte(patchData), patchOpts) + if err != nil { + t.Fatal(err) + } + + if err := retry.UntilSuccess(func() error { + d, err := ds.Get(context.Background(), KmeshDaemonsetName, metav1.GetOptions{}) + if err != nil { + return err + } + if !daemonsetsetComplete(d) { + return fmt.Errorf("rollout is not yet done") + } + return nil + }, retry.Timeout(60*time.Second), retry.Delay(2*time.Second)); err != nil { + t.Fatal("failed to wait for Kmesh rollout status for: %v", err) + } + if _, err := kubetest.CheckPodsAreReady(kubetest.NewPodFetch(t.AllClusters()[0], KmeshNamespace, "app=kmesh")); err != nil { + t.Fatal(err) + } +} + +func daemonsetsetComplete(ds *appsv1.DaemonSet) bool { + return ds.Status.UpdatedNumberScheduled == ds.Status.DesiredNumberScheduled && ds.Status.NumberReady == ds.Status.DesiredNumberScheduled && ds.Status.ObservedGeneration >= ds.Generation +} diff --git a/test/e2e/run_test.sh b/test/e2e/run_test.sh index cf7e57863..05e9025a0 100755 --- a/test/e2e/run_test.sh +++ b/test/e2e/run_test.sh @@ -244,6 +244,9 @@ while (( "$#" )); do IPV6=true shift ;; + --kernel-native) + shift + ;; --cleanup) CLEANUP_KIND=true CLEANUP_REGISTRY=true @@ -285,7 +288,7 @@ if [[ -z "${SKIP_SETUP:-}" ]]; then setup_kmesh fi -cmd="go test -v -tags=integ $ROOT_DIR/test/e2e/... -istio.test.kube.loadbalancer=false ${PARAMS[*]}" +cmd="go test -v -tags=integ $ROOT_DIR/test/e2e/kernel-native/... -istio.test.kube.loadbalancer=false ${PARAMS[*]}" bash -c "$cmd"