feat: support setting multiple TLS certs for different domains on the interceptor proxy (#1116)

Signed-off-by: Jan Wozniak <wozniak.jan@gmail.com>

Author: wozniakjan

CHANGELOG.md

@@ -23,6 +23,7 @@ This changelog keeps track of work items that have been completed and are ready
 
 ### New
 
+- **General**: Support setting multiple TLS certs for different domains on the interceptor proxy ([#1116](https://github.com/kedacore/http-add-on/issues/1116))
 - **General**: TODO ([#TODO](https://github.com/kedacore/http-add-on/issues/TODO))
 
 ### Improvements

Makefile

@@ -45,6 +45,15 @@ DNS.3 = *.interceptor-tls-test-ns
 endef
 export DOMAINS
 
+define ABC_DOMAINS
+basicConstraints=CA:FALSE
+keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = abc
+endef
+export ABC_DOMAINS
+
 # Build targets
 
 build-operator:
@@ -68,6 +77,9 @@ test-certs: rootca-test-certs
 	echo "$$DOMAINS" > certs/domains.ext
 	openssl req -new -nodes -newkey rsa:2048 -keyout certs/tls.key -out certs/tls.csr -subj "/C=US/ST=KedaState/L=KedaCity/O=Keda-Certificates/CN=keda.local"
 	openssl x509 -req -sha256 -days 1024 -in certs/tls.csr -CA certs/RootCA.pem -CAkey certs/RootCA.key -CAcreateserial -extfile certs/domains.ext -out certs/tls.crt
+	echo "$$ABC_DOMAINS" > certs/abc_domains.ext
+	openssl req -new -nodes -newkey rsa:2048 -keyout certs/abc.tls.key -out certs/abc.tls.csr -subj "/C=US/ST=KedaState/L=KedaCity/O=Keda-Certificates/CN=abc"
+	openssl x509 -req -sha256 -days 1024 -in certs/abc.tls.csr -CA certs/RootCA.pem -CAkey certs/RootCA.key -CAcreateserial -extfile certs/abc_domains.ext -out certs/abc.tls.crt
 
 clean-test-certs:
 	rm -r certs || true

config/interceptor/e2e-test/tls/deployment.yaml

@@ -18,13 +18,21 @@ spec:
           value: "/certs/tls.crt"
         - name: KEDA_HTTP_PROXY_TLS_KEY_PATH
           value: "/certs/tls.key"
+        - name: KEDA_HTTP_PROXY_TLS_CERT_STORE_PATHS
+          value: "/additional-certs"
         - name: KEDA_HTTP_PROXY_TLS_PORT
           value: "8443"
         volumeMounts:
           - readOnly: true
             mountPath: "/certs"
             name: certs
+          - readOnly: true
+            mountPath: "/additional-certs/abc-certs"
+            name: abc-certs
       volumes:
         - name: certs
           secret:
             secretName: keda-tls
+        - name: abc-certs
+          secret:
+            secretName: abc-certs

docs/operate.md

@@ -23,3 +23,9 @@ If you need to provide any headers such as authentication details in order to ut
 The interceptor proxy has the ability to run both a HTTP and HTTPS server simultaneously to allow you to scale workloads that use either protocol. By default, the interceptor proxy will only serve over HTTP, but this behavior can be changed by configuring the appropriate environment variables on the deployment.
 
 The TLS server can be enabled by setting the environment variable `KEDA_HTTP_PROXY_TLS_ENABLED` to `true` on the interceptor deployment (`false` by default). The TLS server will start on port `8443` by default, but this can be configured by setting `KEDA_HTTP_PROXY_TLS_PORT` to your desired port number. The TLS server will require valid TLS certificates to start, the path to the certificates can be configured via the `KEDA_HTTP_PROXY_TLS_CERT_PATH` and `KEDA_HTTP_PROXY_TLS_KEY_PATH` environment variables (`/certs/tls.crt` and `/certs/tls.key` by default).
+
+For setting multiple TLS certs, set `KEDA_HTTP_PROXY_TLS_CERT_STORE_PATHS` with comma-separated list of directories that will be recursively searched for any valid cert/key pairs. Currently, two naming patterns are supported
+* `XYZ.crt` + `XYZ.key` - this is a convention when using Kubernetes Secrets of type tls
+* `XYZ.pem` + `XYZ-key.pem`
+
+The matching between certs and requests is performed during the TLS ClientHelo message, where the SNI service name is compared to SANs provided in each cert and the first matching cert will be used for the rest of the TLS handshake.

interceptor/config/serving.go

@@ -41,6 +41,8 @@ type Serving struct {
 	TLSCertPath string `envconfig:"KEDA_HTTP_PROXY_TLS_CERT_PATH" default:"/certs/tls.crt"`
 	// TLSKeyPath is the path to read the private key file from for the TLS server
 	TLSKeyPath string `envconfig:"KEDA_HTTP_PROXY_TLS_KEY_PATH" default:"/certs/tls.key"`
+	// TLSCertStorePaths is a comma separated list of paths to read the certificate/key pairs for the TLS server
+	TLSCertStorePaths string `envconfig:"KEDA_HTTP_PROXY_TLS_CERT_STORE_PATHS" default:""`
 	// TLSPort is the port that the server should serve on if TLS is enabled
 	TLSPort int `envconfig:"KEDA_HTTP_PROXY_TLS_PORT" default:"8443"`
 }

interceptor/main.go

@@ -9,10 +9,13 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/go-logr/logr"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"golang.org/x/exp/maps"
 	"golang.org/x/sync/errgroup"
 	"k8s.io/client-go/kubernetes"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -168,7 +171,7 @@ func main() {
 	// start a proxy server with TLS
 	if proxyTLSEnabled {
 		eg.Go(func() error {
-			proxyTLSConfig := map[string]string{"certificatePath": servingCfg.TLSCertPath, "keyPath": servingCfg.TLSKeyPath}
+			proxyTLSConfig := map[string]string{"certificatePath": servingCfg.TLSCertPath, "keyPath": servingCfg.TLSKeyPath, "certstorePaths": servingCfg.TLSCertStorePaths}
 			proxyTLSPort := servingCfg.TLSPort
 
 			setupLog.Info("starting the proxy server with TLS enabled", "port", proxyTLSPort)
@@ -219,7 +222,7 @@ func runAdminServer(
 
 	addr := fmt.Sprintf("0.0.0.0:%d", port)
 	lggr.Info("admin server starting", "address", addr)
-	return kedahttp.ServeContext(ctx, addr, adminServer, false, nil)
+	return kedahttp.ServeContext(ctx, addr, adminServer, nil)
 }
 
 func runMetricsServer(
@@ -229,7 +232,135 @@ func runMetricsServer(
 ) error {
 	lggr.Info("starting the prometheus metrics server", "port", metricsCfg.OtelPrometheusExporterPort, "path", "/metrics")
 	addr := fmt.Sprintf("0.0.0.0:%d", metricsCfg.OtelPrometheusExporterPort)
-	return kedahttp.ServeContext(ctx, addr, promhttp.Handler(), false, nil)
+	return kedahttp.ServeContext(ctx, addr, promhttp.Handler(), nil)
+}
+
+// addCert adds a certificate to the map of certificates based on the certificate's SANs
+func addCert(m map[string]tls.Certificate, certPath, keyPath string, logger logr.Logger) (*tls.Certificate, error) {
+	cert, err := tls.LoadX509KeyPair(certPath, keyPath)
+	if err != nil {
+		return nil, fmt.Errorf("error loading certificate and key: %w", err)
+	}
+	if cert.Leaf == nil {
+		if len(cert.Certificate) == 0 {
+			return nil, fmt.Errorf("no certificate found in certificate chain")
+		}
+		cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
+		if err != nil {
+			return nil, fmt.Errorf("error parsing certificate: %w", err)
+		}
+	}
+	for _, d := range cert.Leaf.DNSNames {
+		logger.Info("adding certificate", "dns", d)
+		m[d] = cert
+	}
+	for _, ip := range cert.Leaf.IPAddresses {
+		logger.Info("adding certificate", "ip", ip.String())
+		m[ip.String()] = cert
+	}
+	for _, uri := range cert.Leaf.URIs {
+		logger.Info("adding certificate", "uri", uri.String())
+		m[uri.String()] = cert
+	}
+	return &cert, nil
+}
+
+func defaultCertPool(logger logr.Logger) *x509.CertPool {
+	systemCAs, err := x509.SystemCertPool()
+	if err == nil {
+		return systemCAs
+	}
+
+	logger.Info("error loading system CA pool, using empty pool", "error", err)
+	return x509.NewCertPool()
+}
+
+// getTLSConfig creates a TLS config from KEDA_HTTP_PROXY_TLS_CERT_PATH, KEDA_HTTP_PROXY_TLS_KEY_PATH and KEDA_HTTP_PROXY_TLS_CERTSTORE_PATHS
+// The matching between request and certificate is performed by comparing TLS/SNI server name with x509 SANs
+func getTLSConfig(tlsConfig map[string]string, logger logr.Logger) (*tls.Config, error) {
+	certPath := tlsConfig["certificatePath"]
+	keyPath := tlsConfig["keyPath"]
+	certStorePaths := tlsConfig["certstorePaths"]
+	servingTLS := &tls.Config{
+		RootCAs: defaultCertPool(logger),
+	}
+	var defaultCert *tls.Certificate
+
+	uriDomainsToCerts := make(map[string]tls.Certificate)
+	if certPath != "" && keyPath != "" {
+		cert, err := addCert(uriDomainsToCerts, certPath, keyPath, logger)
+		if err != nil {
+			return servingTLS, fmt.Errorf("error adding certificate and key: %w", err)
+		}
+		defaultCert = cert
+		rawCert, err := os.ReadFile(certPath)
+		if err != nil {
+			return servingTLS, fmt.Errorf("error reading certificate: %w", err)
+		}
+		servingTLS.RootCAs.AppendCertsFromPEM(rawCert)
+	}
+
+	if certStorePaths != "" {
+		certFiles := make(map[string]string)
+		keyFiles := make(map[string]string)
+		dirPaths := strings.Split(certStorePaths, ",")
+		for _, dir := range dirPaths {
+			err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+				if err != nil {
+					return err
+				}
+				if info.IsDir() {
+					return nil
+				}
+				switch {
+				case strings.HasSuffix(path, "-key.pem"):
+					certID := path[:len(path)-8]
+					keyFiles[certID] = path
+				case strings.HasSuffix(path, ".pem"):
+					certID := path[:len(path)-4]
+					certFiles[certID] = path
+				case strings.HasSuffix(path, ".key"):
+					certID := path[:len(path)-4]
+					keyFiles[certID] = path
+				case strings.HasSuffix(path, ".crt"):
+					certID := path[:len(path)-4]
+					certFiles[certID] = path
+				}
+				return nil
+			})
+			if err != nil {
+				return servingTLS, fmt.Errorf("error walking certificate store: %w", err)
+			}
+		}
+
+		for certID, certPath := range certFiles {
+			logger.Info("adding certificate", "certID", certID, "certPath", certPath)
+			keyPath, ok := keyFiles[certID]
+			if !ok {
+				return servingTLS, fmt.Errorf("no key found for certificate %s", certPath)
+			}
+			if _, err := addCert(uriDomainsToCerts, certPath, keyPath, logger); err != nil {
+				return servingTLS, fmt.Errorf("error adding certificate %s: %w", certPath, err)
+			}
+			rawCert, err := os.ReadFile(certPath)
+			if err != nil {
+				return servingTLS, fmt.Errorf("error reading certificate: %w", err)
+			}
+			servingTLS.RootCAs.AppendCertsFromPEM(rawCert)
+		}
+	}
+
+	servingTLS.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+		if cert, ok := uriDomainsToCerts[hello.ServerName]; ok {
+			return &cert, nil
+		}
+		if defaultCert != nil {
+			return defaultCert, nil
+		}
+		return nil, fmt.Errorf("no certificate found for %s", hello.ServerName)
+	}
+	servingTLS.Certificates = maps.Values(uriDomainsToCerts)
+	return servingTLS, nil
 }
 
 func runProxyServer(
@@ -251,33 +382,29 @@ func runProxyServer(
 	})
 	go probeHandler.Start(ctx)
 
-	tlsCfg := tls.Config{}
+	var tlsCfg *tls.Config
 	if tlsEnabled {
-		caCert, err := os.ReadFile(tlsConfig["certificatePath"])
-		if err != nil {
-			logger.Error(fmt.Errorf("error reading file from TLSCertPath"), "error", err)
-			os.Exit(1)
-		}
-		caCertPool := x509.NewCertPool()
-		caCertPool.AppendCertsFromPEM(caCert)
-		cert, err := tls.LoadX509KeyPair(tlsConfig["certificatePath"], tlsConfig["keyPath"])
-
+		cfg, err := getTLSConfig(tlsConfig, logger)
 		if err != nil {
-			logger.Error(fmt.Errorf("error creating TLS configuration for proxy server"), "error", err)
+			logger.Error(fmt.Errorf("error creating certGetter for proxy server"), "error", err)
 			os.Exit(1)
 		}
-
-		tlsCfg.RootCAs = caCertPool
-		tlsCfg.Certificates = []tls.Certificate{cert}
+		tlsCfg = cfg
 	}
 
 	var upstreamHandler http.Handler
+	forwardingTLSCfg := &tls.Config{}
+	if tlsCfg != nil {
+		forwardingTLSCfg.RootCAs = tlsCfg.RootCAs
+		forwardingTLSCfg.Certificates = tlsCfg.Certificates
+	}
+
 	upstreamHandler = newForwardingHandler(
 		logger,
 		dialContextFunc,
 		waitFunc,
 		newForwardingConfigFromTimeouts(timeouts),
-		&tlsCfg,
+		forwardingTLSCfg,
 	)
 	upstreamHandler = middleware.NewCountingMiddleware(
 		q,
@@ -302,5 +429,8 @@ func runProxyServer(
 
 	addr := fmt.Sprintf("0.0.0.0:%d", port)
 	logger.Info("proxy server starting", "address", addr)
-	return kedahttp.ServeContext(ctx, addr, rootHandler, tlsEnabled, tlsConfig)
+	if tlsEnabled {
+		return kedahttp.ServeContext(ctx, addr, rootHandler, tlsCfg)
+	}
+	return kedahttp.ServeContext(ctx, addr, rootHandler, nil)
 }