[release-1.10] ✨ Add mTLS support to runtime extension server and client (#12519)

* Add mTLS support to runtime extension server and client

* Fix unit test for release-1.10

---------

Co-authored-by: fabriziopandini <fpandini@vmware.com>
Co-authored-by: Stefan Bueringer <buringerst@vmware.com>

Author: 3 people

exp/runtime/server/server.go

@@ -80,6 +80,10 @@ type Options struct {
 	// Note: This option is only used when TLSOpts does not set GetCertificate.
 	KeyName string
 
+	// ClientCAName is the CA certificate name which server used to verify remote(client)'s certificate.
+	// Defaults to "", which means server does not verify client's certificate.
+	ClientCAName string
+
 	// TLSOpts is used to allow configuring the TLS config used for the server.
 	// This also allows providing a certificate via GetCertificate.
 	TLSOpts []func(*tls.Config)
@@ -105,13 +109,14 @@ func New(options Options) (*Server, error) {
 
 	webhookServer := webhook.NewServer(
 		webhook.Options{
-			Port:       options.Port,
-			Host:       options.Host,
-			CertDir:    options.CertDir,
-			CertName:   options.CertName,
-			KeyName:    options.KeyName,
-			TLSOpts:    options.TLSOpts,
-			WebhookMux: http.NewServeMux(),
+			Port:         options.Port,
+			Host:         options.Host,
+			ClientCAName: options.ClientCAName,
+			CertDir:      options.CertDir,
+			CertName:     options.CertName,
+			KeyName:      options.KeyName,
+			TLSOpts:      options.TLSOpts,
+			WebhookMux:   http.NewServeMux(),
 		},
 	)
 

internal/runtime/client/client.go

@@ -60,6 +60,8 @@ const defaultDiscoveryTimeout = 10 * time.Second
 
 // Options are creation options for a Client.
 type Options struct {
+	CertFile string // Path of the PEM-encoded client certificate.
+	KeyFile  string // Path of the PEM-encoded client key.
 	Catalog  *runtimecatalog.Catalog
 	Registry runtimeregistry.ExtensionRegistry
 	Client   ctrlclient.Client
@@ -68,6 +70,8 @@ type Options struct {
 // New returns a new Client.
 func New(options Options) runtimeclient.Client {
 	return &client{
+		certFile: options.CertFile,
+		keyFile:  options.KeyFile,
 		catalog:  options.Catalog,
 		registry: options.Registry,
 		client:   options.Client,
@@ -77,6 +81,8 @@ func New(options Options) runtimeclient.Client {
 var _ runtimeclient.Client = &client{}
 
 type client struct {
+	certFile string
+	keyFile  string
 	catalog  *runtimecatalog.Catalog
 	registry runtimeregistry.ExtensionRegistry
 	client   ctrlclient.Client
@@ -102,6 +108,8 @@ func (c *client) Discover(ctx context.Context, extensionConfig *runtimev1.Extens
 	request := &runtimehooksv1.DiscoveryRequest{}
 	response := &runtimehooksv1.DiscoveryResponse{}
 	opts := &httpCallOptions{
+		certFile:        c.certFile,
+		keyFile:         c.keyFile,
 		catalog:         c.catalog,
 		config:          extensionConfig.Spec.ClientConfig,
 		registrationGVH: hookGVH,
@@ -329,6 +337,8 @@ func (c *client) CallExtension(ctx context.Context, hook runtimecatalog.Hook, fo
 	}
 
 	httpOpts := &httpCallOptions{
+		certFile:        c.certFile,
+		keyFile:         c.keyFile,
 		catalog:         c.catalog,
 		config:          registration.ClientConfig,
 		registrationGVH: registration.GroupVersionHook,
@@ -396,6 +406,8 @@ func cloneAndAddSettings(request runtimehooksv1.RequestObject, registrationSetti
 }
 
 type httpCallOptions struct {
+	certFile        string
+	keyFile         string
 	catalog         *runtimecatalog.Catalog
 	config          runtimev1.ClientConfig
 	registrationGVH runtimecatalog.GroupVersionHook
@@ -484,6 +496,8 @@ func httpCall(ctx context.Context, request, response runtime.Object, opts *httpC
 	client := http.DefaultClient
 	tlsConfig, err := transport.TLSConfigFor(&transport.Config{
 		TLS: transport.TLSConfig{
+			CertFile:   opts.certFile,
+			KeyFile:    opts.keyFile,
 			CAData:     opts.config.CABundle,
 			ServerName: extensionURL.Hostname(),
 		},

internal/runtime/client/client_test.go

@@ -19,10 +19,13 @@ package client
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"path/filepath"
 	"regexp"
 	"testing"
 
@@ -842,6 +845,102 @@ func TestClient_CallExtension(t *testing.T) {
 	}
 }
 
+func TestClient_CallExtensionWithClientAuthentication(t *testing.T) {
+	ns := &corev1.Namespace{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Namespace",
+			APIVersion: corev1.SchemeGroupVersion.String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "foo",
+		},
+	}
+
+	validExtensionHandlerWithFailPolicy := runtimev1.ExtensionConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			ResourceVersion: "15",
+		},
+		Spec: runtimev1.ExtensionConfigSpec{
+			ClientConfig: runtimev1.ClientConfig{
+				// Set a fake URL, in test cases where we start the test server the URL will be overridden.
+				URL:      ptr.To("https://127.0.0.1/"),
+				CABundle: testcerts.CACert,
+			},
+			NamespaceSelector: &metav1.LabelSelector{},
+		},
+		Status: runtimev1.ExtensionConfigStatus{
+			Handlers: []runtimev1.ExtensionHandler{
+				{
+					Name: "valid-extension",
+					RequestHook: runtimev1.GroupVersionHook{
+						APIVersion: fakev1alpha1.GroupVersion.String(),
+						Hook:       "FakeHook",
+					},
+					TimeoutSeconds: ptr.To[int32](1),
+					FailurePolicy:  ptr.To(runtimev1.FailurePolicyFail),
+				},
+			},
+		},
+	}
+
+	g := NewWithT(t)
+
+	tmpDir := t.TempDir()
+	clientCertFile := filepath.Join(tmpDir, "tls.crt")
+	g.Expect(os.WriteFile(clientCertFile, testcerts.ClientCert, 0600)).To(Succeed())
+	clientKeyFile := filepath.Join(tmpDir, "tls.key")
+	g.Expect(os.WriteFile(clientKeyFile, testcerts.ClientKey, 0600)).To(Succeed())
+
+	var serverCallCount int
+	srv := createSecureTestServer(testServerConfig{
+		start: true,
+		responses: map[string]testServerResponse{
+			"/*": response(runtimehooksv1.ResponseStatusSuccess),
+		},
+	}, func() {
+		serverCallCount++
+	})
+
+	// Setup the runtime extension server so it requires client authentication with certificates signed by a given CA.
+	certpool := x509.NewCertPool()
+	certpool.AppendCertsFromPEM(testcerts.CACert)
+	srv.TLS.ClientAuth = tls.RequireAndVerifyClientCert
+	srv.TLS.ClientCAs = certpool
+
+	srv.StartTLS()
+	defer srv.Close()
+
+	// Set the URL to the real address of the test server.
+	validExtensionHandlerWithFailPolicy.Spec.ClientConfig.URL = ptr.To(fmt.Sprintf("https://%s/", srv.Listener.Addr().String()))
+
+	cat := runtimecatalog.New()
+	_ = fakev1alpha1.AddToCatalog(cat)
+	_ = fakev1alpha2.AddToCatalog(cat)
+	fakeClient := fake.NewClientBuilder().
+		WithObjects(ns).
+		Build()
+
+	c := New(Options{
+		// Add client authentication credentials to the client
+		CertFile: clientCertFile,
+		KeyFile:  clientKeyFile,
+		Catalog:  cat,
+		Registry: registry([]runtimev1.ExtensionConfig{validExtensionHandlerWithFailPolicy}),
+		Client:   fakeClient,
+	})
+
+	obj := &clusterv1.Cluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "cluster",
+			Namespace: "foo",
+		},
+	}
+	// Call once without caching.
+	err := c.CallExtension(context.Background(), fakev1alpha1.FakeHook, obj, "valid-extension", &fakev1alpha1.FakeRequest{}, &fakev1alpha1.FakeResponse{})
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(serverCallCount).To(Equal(1))
+}
+
 func cacheKeyFunc(extensionName, extensionConfigResourceVersion string, request runtimehooksv1.RequestObject) string {
 	// Note: extensionName is identical to the value of the name parameter passed into CallExtension.
 	s := fmt.Sprintf("%s-%s", extensionName, extensionConfigResourceVersion)

main.go

@@ -110,6 +110,8 @@ var (
 	webhookCertDir              string
 	webhookCertName             string
 	webhookKeyName              string
+	runtimeExtensionCertFile    string
+	runtimeExtensionKeyFile     string
 	healthAddr                  string
 	managerOptions              = flags.ManagerOptions{}
 	logOptions                  = logs.NewOptions()
@@ -259,10 +261,16 @@ func InitFlags(fs *pflag.FlagSet) {
 		"Webhook cert dir.")
 
 	fs.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt",
-		"Webhook cert name.")
+		"Name of the file for webhook's server certificate; the file must be placed under webhook-cert-dir.")
 
 	fs.StringVar(&webhookKeyName, "webhook-key-name", "tls.key",
-		"Webhook key name.")
+		"Name of the file for webhook's server key; the file must be placed under webhook-cert-dir.")
+
+	fs.StringVar(&runtimeExtensionCertFile, "runtime-extension-client-cert-file", "",
+		"Path of the PEM-encoded client certificate to be used when calling runtime extensions.")
+
+	fs.StringVar(&runtimeExtensionKeyFile, "runtime-extension-client-key-file", "",
+		"Path of the PEM-encoded client key to be used when calling runtime extensions.")
 
 	fs.StringVar(&healthAddr, "health-addr", ":9440",
 		"The address the health endpoint binds to.")
@@ -531,6 +539,8 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager, watchNamespaces map
 	if feature.Gates.Enabled(feature.RuntimeSDK) {
 		// This is the creation of the runtimeClient for the controllers, embedding a shared catalog and registry instance.
 		runtimeClient = internalruntimeclient.New(internalruntimeclient.Options{
+			CertFile: runtimeExtensionCertFile,
+			KeyFile:  runtimeExtensionKeyFile,
 			Catalog:  catalog,
 			Registry: runtimeregistry.New(),
 			Client:   mgr.GetClient(),