Add E2E test framework for Gateway API

Author: zac-nixon

test/e2e/gateway/alb_instance_target.go

@@ -7,6 +7,7 @@ import (
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
 	"sigs.k8s.io/aws-load-balancer-controller/test/framework"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	gwv1 "sigs.k8s.io/gateway-api/apis/v1"
 )
 
 type ALBInstanceTestStack struct {
@@ -17,7 +18,7 @@ func (s *ALBInstanceTestStack) Deploy(ctx context.Context, f *framework.Framewor
 	dp := buildDeploymentSpec(f.Options.TestImageRegistry)
 	svc := buildServiceSpec()
 	gwc := buildGatewayClassSpec("gateway.k8s.aws/alb")
-	gw := buildGatewaySpec(gwc)
+	gw := buildBasicGatewaySpec(gwc, gwv1.HTTPProtocolType)
 	lbc := buildLoadBalancerConfig(spec)
 	httpr := buildHTTPRoute()
 	s.albResourceStack = newALBResourceStack(dp, svc, gwc, gw, lbc, httpr, "service-instance-e2e", false)
@@ -30,6 +31,7 @@ func (s *ALBInstanceTestStack) ScaleDeployment(ctx context.Context, f *framework
 }
 
 func (s *ALBInstanceTestStack) Cleanup(ctx context.Context, f *framework.Framework) {
+	_ = f.K8sClient.Delete(ctx, s.albResourceStack.httpr)
 	s.albResourceStack.Cleanup(ctx, f)
 }
 

test/e2e/gateway/alb_instance_target_test.go

@@ -19,6 +19,9 @@ var _ = Describe("test k8s alb gateway reconciled by the aws load balancer contr
 		lbARN   string
 	)
 	BeforeEach(func() {
+		if !tf.Options.EnableGatewayTests {
+			Skip("Skipping gateway tests")
+		}
 		ctx = context.Background()
 		stack = ALBInstanceTestStack{}
 	})
@@ -62,6 +65,7 @@ var _ = Describe("test k8s alb gateway reconciled by the aws load balancer contr
 					TargetGroupHC: &verifier.TargetGroupHC{
 						Protocol:           "HTTP",
 						Port:               "traffic-port",
+						Path:               "/",
 						Interval:           15,
 						Timeout:            5,
 						HealthyThreshold:   3,

test/e2e/gateway/alb_resource_stack.go

@@ -53,7 +53,16 @@ func (s *albResourceStack) getListenersPortMap() map[string]string {
 }
 
 func (s *albResourceStack) getTargetGroupNodePortMap() map[string]string {
-	return s.commonStack.getTargetGroupNodePortMap()
+	res := s.commonStack.getTargetGroupNodePortMap()
+
+	for p := range res {
+		// TODO - kinda a hack to get HTTP to work.
+		if res[p] == string(corev1.ProtocolTCP) {
+			res[p] = "HTTP"
+		}
+	}
+
+	return res
 }
 
 func (s *albResourceStack) getHealthCheckNodePort() string {

test/e2e/gateway/common_resource_stack.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
 	elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1"
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/algorithm"
@@ -107,8 +108,6 @@ func (s *commonResourceStack) ScaleDeployment(ctx context.Context, f *framework.
 
 func (s *commonResourceStack) Cleanup(ctx context.Context, f *framework.Framework) {
 	_ = s.deleteGateway(ctx, f)
-	// todo - fix
-	time.Sleep(5 * time.Minute)
 	_ = s.deleteNamespace(ctx, f)
 	_ = s.deleteGatewayClass(ctx, f)
 }
@@ -240,7 +239,20 @@ func (s *commonResourceStack) deleteService(ctx context.Context, f *framework.Fr
 }
 
 func (s *commonResourceStack) deleteGateway(ctx context.Context, f *framework.Framework) error {
-	return f.K8sClient.Delete(ctx, s.gw)
+	err := f.K8sClient.Delete(ctx, s.gw)
+	if err != nil {
+		return err
+	}
+	observedGW := &gwv1.Gateway{}
+	return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		if err := f.K8sClient.Get(ctx, k8s.NamespacedName(s.gw), observedGW); err != nil {
+			if apierrs.IsNotFound(err) {
+				return true, nil
+			}
+			return false, err
+		}
+		return false, nil
+	}, ctx.Done())
 }
 
 func (s *commonResourceStack) deleteGatewayClass(ctx context.Context, f *framework.Framework) error {

test/e2e/gateway/nlb_instance_target.go

@@ -7,6 +7,7 @@ import (
 	"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
 	"sigs.k8s.io/aws-load-balancer-controller/test/framework"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	gwv1 "sigs.k8s.io/gateway-api/apis/v1"
 )
 
 type NLBInstanceTestStack struct {
@@ -17,7 +18,7 @@ func (s *NLBInstanceTestStack) Deploy(ctx context.Context, f *framework.Framewor
 	dp := buildDeploymentSpec(f.Options.TestImageRegistry)
 	svc := buildServiceSpec()
 	gwc := buildGatewayClassSpec("gateway.k8s.aws/nlb")
-	gw := buildGatewaySpec(gwc)
+	gw := buildBasicGatewaySpec(gwc, gwv1.TCPProtocolType)
 	lbc := buildLoadBalancerConfig(spec)
 	tcpr := buildTCPRoute()
 	s.nlbResourceStack = newNLBResourceStack(dp, svc, gwc, gw, lbc, tcpr, "service-instance-e2e", false)
@@ -30,6 +31,7 @@ func (s *NLBInstanceTestStack) ScaleDeployment(ctx context.Context, f *framework
 }
 
 func (s *NLBInstanceTestStack) Cleanup(ctx context.Context, f *framework.Framework) {
+	_ = f.K8sClient.Delete(ctx, s.nlbResourceStack.tcpr)
 	s.nlbResourceStack.Cleanup(ctx, f)
 }
 

test/e2e/gateway/nlb_instance_target_test.go

@@ -19,6 +19,9 @@ var _ = Describe("test nlb gateway reconciled by the aws load balancer controlle
 		lbARN   string
 	)
 	BeforeEach(func() {
+		if !tf.Options.EnableGatewayTests {
+			Skip("Skipping gateway tests")
+		}
 		ctx = context.Background()
 		stack = NLBInstanceTestStack{}
 	})

test/e2e/gateway/shared_resource_definitions.go

@@ -98,7 +98,7 @@ func buildLoadBalancerConfig(spec elbv2gw.LoadBalancerConfigurationSpec) *elbv2g
 	return lbc
 }
 
-func buildGatewaySpec(gwc *gwv1.GatewayClass) *gwv1.Gateway {
+func buildBasicGatewaySpec(gwc *gwv1.GatewayClass, protocol gwv1.ProtocolType) *gwv1.Gateway {
 	gw := &gwv1.Gateway{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: defaultName,
@@ -109,7 +109,7 @@ func buildGatewaySpec(gwc *gwv1.GatewayClass) *gwv1.Gateway {
 				{
 					Name:     "test-listener",
 					Port:     80,
-					Protocol: gwv1.TCPProtocolType,
+					Protocol: protocol,
 				},
 			},
 			Infrastructure: &gwv1.GatewayInfrastructure{

test/e2e/ingress/ingress_suite_test.go

@@ -1,12 +1,10 @@
 package ingress
 
 import (
-	"fmt"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	framework "sigs.k8s.io/aws-load-balancer-controller/test/framework"
 	"testing"
-	"time"
 )
 
 var tf *framework.Framework
@@ -17,22 +15,9 @@ func TestIngress(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-
-	fmt.Println("before!")
 	var err error
 
-	defer func() {
-		fmt.Println("Hello!?")
-		if r := recover(); r != nil {
-			fmt.Println("Recovered in f", r)
-		}
-	}()
-
 	tf, err = framework.InitFramework()
 
-	fmt.Println("after!")
-
-	time.Sleep(10 * time.Second)
-
 	Expect(err).NotTo(HaveOccurred())
 })