Add support to provide TLS for Gateway (#3900)

Problem: Users want to be able to specify their gateway's identity when communicating to the backend pods.

Solution: Add a field to provide a secret name that stores that gateways cert and key to be used when doing TLS handshake with backend pods.

Author: salonichf5

internal/controller/manager.go

@@ -140,9 +140,10 @@ func StartManager(cfg config.Config) error {
 			GenericValidator:    genericValidator,
 			PolicyValidator:     policyManager,
 		},
-		EventRecorder:  recorder,
-		MustExtractGVK: mustExtractGVK,
-		PlusSecrets:    plusSecrets,
+		EventRecorder:        recorder,
+		MustExtractGVK:       mustExtractGVK,
+		PlusSecrets:          plusSecrets,
+		ExperimentalFeatures: cfg.ExperimentalFeatures,
 	})
 
 	var handlerCollector handlerMetricsCollector = collectors.NewControllerNoopCollector()

internal/controller/nginx/config/base_http_config.go

@@ -19,6 +19,7 @@ type AccessLog struct {
 type httpConfig struct {
 	DNSResolver             *dataplane.DNSResolverConfig
 	AccessLog               *AccessLog
+	GatewaySecretID         dataplane.SSLKeyPairID
 	Includes                []shared.Include
 	NginxReadinessProbePort int32
 	IPFamily                shared.IPFamily
@@ -35,6 +36,7 @@ func executeBaseHTTPConfig(conf dataplane.Configuration) []executeResult {
 		IPFamily:                getIPFamily(conf.BaseHTTPConfig),
 		DNSResolver:             conf.BaseHTTPConfig.DNSResolver,
 		AccessLog:               buildAccessLog(conf.Logging.AccessLog),
+		GatewaySecretID:         conf.BaseHTTPConfig.GatewaySecretID,
 	}
 
 	results := make([]executeResult, 0, len(includes)+1)

internal/controller/nginx/config/base_http_config_template.go

@@ -61,6 +61,12 @@ access_log {{ .AccessLog.Path }} {{ .AccessLog.FormatName }};
 {{- end }}
 {{- end }}
 
+{{- if $.GatewaySecretID }}
+# Gateway Certificate
+proxy_ssl_certificate /etc/nginx/secrets/{{ $.GatewaySecretID }}.pem;
+proxy_ssl_certificate_key /etc/nginx/secrets/{{ $.GatewaySecretID }}.pem;
+{{- end }}
+
 {{ range $i := .Includes -}}
 include {{ $i.Name }};
 {{ end -}}

internal/controller/nginx/config/base_http_config_test.go

@@ -356,3 +356,49 @@ func TestExecuteBaseHttp_DNSResolver(t *testing.T) {
 		})
 	}
 }
+
+func TestExecuteBaseHttp_GatewaySecretID(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name           string
+		expectedConfig string
+		conf           dataplane.Configuration
+	}{
+		{
+			name: "with GatewaySecretID",
+			conf: dataplane.Configuration{
+				BaseHTTPConfig: dataplane.BaseHTTPConfig{
+					GatewaySecretID: "client-secret",
+				},
+			},
+			expectedConfig: "proxy_ssl_certificate /etc/nginx/secrets/client-secret.pem;" +
+				"\nproxy_ssl_certificate_key /etc/nginx/secrets/client-secret.pem;",
+		},
+		{
+			name: "without GatewaySecretID",
+			conf: dataplane.Configuration{
+				BaseHTTPConfig: dataplane.BaseHTTPConfig{
+					GatewaySecretID: "",
+				},
+			},
+			expectedConfig: "",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			g := NewWithT(t)
+
+			res := executeBaseHTTPConfig(test.conf)
+			g.Expect(res).To(HaveLen(1))
+
+			httpConfig := string(res[0].data)
+
+			if test.expectedConfig != "" {
+				g.Expect(httpConfig).To(ContainSubstring(test.expectedConfig))
+			}
+		})
+	}
+}

internal/controller/state/conditions/conditions.go

@@ -146,6 +146,14 @@ const (
 	// parametersRef resource is invalid.
 	GatewayReasonParamsRefInvalid v1.GatewayConditionReason = "ParametersRefInvalid"
 
+	// GatewayReasonSecretRefInvalid is used with the "GatewayResolvedRefs" condition when the
+	// secretRef resource is invalid.
+	GatewayReasonSecretRefInvalid v1.GatewayConditionReason = "SecretRefInvalid"
+
+	// GatewayReasonSecretRefNotPermitted is used with the "GatewayResolvedRefs" condition when the
+	// secretRef resource is not permitted by any ReferenceGrant.
+	GatewayReasonSecretRefNotPermitted v1.GatewayConditionReason = "SecretRefNotPermitted"
+
 	// PolicyReasonAncestorLimitReached is used with the "PolicyAccepted" condition when a policy
 	// cannot be applied because the ancestor status list has reached the maximum size of 16.
 	PolicyReasonAncestorLimitReached v1.PolicyConditionReason = "AncestorLimitReached"
@@ -292,6 +300,27 @@ func NewGatewayClassUnsupportedVersion(recommendedVersion string) []Condition {
 	}
 }
 
+// NewGatewaySecretRefNotPermitted returns Condition that indicates that the Gateway references a TLS secret that is not
+// permitted by any ReferenceGrant.
+func NewGatewaySecretRefNotPermitted(msg string) Condition {
+	return Condition{
+		Type:    string(GatewayReasonResolvedRefs),
+		Status:  metav1.ConditionFalse,
+		Reason:  string(GatewayReasonSecretRefNotPermitted),
+		Message: msg,
+	}
+}
+
+// NewGatewaySecretRefInvalid returns Condition that indicates that the Gateway references a TLS secret that is invalid.
+func NewGatewaySecretRefInvalid(msg string) Condition {
+	return Condition{
+		Type:    string(GatewayReasonResolvedRefs),
+		Status:  metav1.ConditionFalse,
+		Reason:  string(GatewayReasonSecretRefInvalid),
+		Message: msg,
+	}
+}
+
 // NewGatewayClassConflict returns a Condition that indicates that the GatewayClass is not accepted
 // due to a conflict with another GatewayClass.
 func NewGatewayClassConflict() Condition {
@@ -846,6 +875,25 @@ func NewGatewayInvalid(msg string) []Condition {
 	}
 }
 
+// NewGatewayUnsupportedValue returns Conditions that indicate that a field of the Gateway has an unsupported value.
+// Unsupported means that the value is not supported by the implementation under certain conditions or invalid.
+func NewGatewayUnsupportedValue(msg string) []Condition {
+	return []Condition{
+		{
+			Type:    string(v1.GatewayConditionAccepted),
+			Status:  metav1.ConditionFalse,
+			Reason:  string(GatewayReasonUnsupportedValue),
+			Message: msg,
+		},
+		{
+			Type:    string(v1.GatewayConditionProgrammed),
+			Status:  metav1.ConditionFalse,
+			Reason:  string(GatewayReasonUnsupportedValue),
+			Message: msg,
+		},
+	}
+}
+
 // NewGatewayUnsupportedAddress returns a Condition that indicates the Gateway is not accepted because it
 // contains an address type that is not supported.
 func NewGatewayUnsupportedAddress(msg string) Condition {

internal/controller/state/dataplane/configuration.go

@@ -85,9 +85,10 @@ func BuildConfiguration(
 			gateway,
 			serviceResolver,
 			g.ReferencedServices,
-			baseHTTPConfig.IPFamily),
+			baseHTTPConfig.IPFamily,
+		),
 		BackendGroups: backendGroups,
-		SSLKeyPairs:   buildSSLKeyPairs(g.ReferencedSecrets, gateway.Listeners),
+		SSLKeyPairs:   buildSSLKeyPairs(g.ReferencedSecrets, gateway),
 		CertBundles: buildCertBundles(
 			buildRefCertificateBundles(g.ReferencedSecrets, g.ReferencedCaCertConfigMaps),
 			backendGroups,
@@ -252,14 +253,14 @@ func buildStreamUpstreams(
 }
 
 // buildSSLKeyPairs builds the SSLKeyPairs from the Secrets. It will only include Secrets that are referenced by
-// valid listeners, so that we don't include unused Secrets in the configuration of the data plane.
+// valid gateway and its listeners, so that we don't include unused Secrets in the configuration of the data plane.
 func buildSSLKeyPairs(
 	secrets map[types.NamespacedName]*graph.Secret,
-	listeners []*graph.Listener,
+	gateway *graph.Gateway,
 ) map[SSLKeyPairID]SSLKeyPair {
 	keyPairs := make(map[SSLKeyPairID]SSLKeyPair)
 
-	for _, l := range listeners {
+	for _, l := range gateway.Listeners {
 		if l.Valid && l.ResolvedSecret != nil {
 			id := generateSSLKeyPairID(*l.ResolvedSecret)
 			secret := secrets[*l.ResolvedSecret]
@@ -272,6 +273,15 @@ func buildSSLKeyPairs(
 		}
 	}
 
+	if gateway.Valid && gateway.SecretRef != nil {
+		id := generateSSLKeyPairID(*gateway.SecretRef)
+		secret := secrets[*gateway.SecretRef]
+		keyPairs[id] = SSLKeyPair{
+			Cert: secret.CertBundle.Cert.TLSCert,
+			Key:  secret.CertBundle.Cert.TLSPrivateKey,
+		}
+	}
+
 	return keyPairs
 }
 
@@ -1058,6 +1068,10 @@ func buildBaseHTTPConfig(
 		NginxReadinessProbePort: DefaultNginxReadinessProbePort,
 	}
 
+	if gateway.Valid && gateway.SecretRef != nil {
+		baseConfig.GatewaySecretID = generateSSLKeyPairID(*gateway.SecretRef)
+	}
+
 	// safe to access EffectiveNginxProxy since we only call this function when the Gateway is not nil.
 	np := gateway.EffectiveNginxProxy
 	if np == nil {
@@ -1081,8 +1095,20 @@ func buildBaseHTTPConfig(
 		}
 	}
 
+	if port := getNginxReadinessProbePort(np); port != 0 {
+		baseConfig.NginxReadinessProbePort = port
+	}
+
 	baseConfig.RewriteClientIPSettings = buildRewriteClientIPConfig(np.RewriteClientIP)
 
+	baseConfig.DNSResolver = buildDNSResolverConfig(np.DNSResolver)
+
+	return baseConfig
+}
+
+func getNginxReadinessProbePort(np *graph.EffectiveNginxProxy) int32 {
+	var port int32
+
 	if np.Kubernetes != nil {
 		var containerSpec *ngfAPIv1alpha2.ContainerSpec
 		if np.Kubernetes.Deployment != nil {
@@ -1091,13 +1117,10 @@ func buildBaseHTTPConfig(
 			containerSpec = &np.Kubernetes.DaemonSet.Container
 		}
 		if containerSpec != nil && containerSpec.ReadinessProbe != nil && containerSpec.ReadinessProbe.Port != nil {
-			baseConfig.NginxReadinessProbePort = *containerSpec.ReadinessProbe.Port
+			port = *containerSpec.ReadinessProbe.Port
 		}
 	}
-
-	baseConfig.DNSResolver = buildDNSResolverConfig(np.DNSResolver)
-
-	return baseConfig
+	return port
 }
 
 // buildBaseStreamConfig generates the base stream context config that should be applied to all stream servers.