EXPERIMENTAL HA support

This change lets the HNC manager be run both without webhooks and
in webhook-only mode. This allows us to create two Deployments - one for
the controller which actually makes changes, and another replicated
deployment for the webhooks, each of which independently sync their
in-memory forests and serve webhook requests.

Taking advantage of this new feature requires considerable changes to
the manifests, which I'm checking in as a separate commit.

Tested: change passes smoke test when deployed with HA manifests, but I
haven't done any careful validation that (for example) the HA deployment
isn't actually writing anything.

Author: adrianludwin

cmd/manager/main.go

@@ -35,6 +35,7 @@ import (
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 
@@ -59,7 +60,7 @@ var (
 	maxReconciles           int
 	enableLeaderElection    bool
 	leaderElectionId        string
-	novalidation            bool
+	noWebhooks              bool
 	debugLogs               bool
 	testLog                 bool
 	internalCert            bool
@@ -71,6 +72,7 @@ var (
 	managedNamespaceLabels  arrayArg
 	managedNamespaceAnnots  arrayArg
 	includedNamespacesRegex string
+	webhooksOnly            bool
 )
 
 // init preloads some global vars before main() starts. Since this is the top-level module, I'm not
@@ -93,12 +95,19 @@ func main() {
 	defer metricsCleanupFn()
 	mgr := createManager()
 
-	// Make sure certs are generated and valid if webhooks are enabled and internal certs are used.
-	setupLog.Info("Starting certificate generation")
-	certsReady, err := setup.CreateCertsIfNeeded(mgr, novalidation, internalCert, restartOnSecretRefresh)
-	if err != nil {
-		setupLog.Error(err, "unable to set up cert rotation")
-		os.Exit(1)
+	// Make sure certs are managed if requested. In webhooks-only mode, we don't run the manager, and
+	// rely on either a controller running in a different HNC deployment, or an external tool such as
+	// cert-manager.
+	certsReady := make(chan struct{})
+	if internalCert && !webhooksOnly {
+		setupLog.Info("Starting certificate generation")
+		err := setup.ManageCerts(mgr, certsReady, restartOnSecretRefresh)
+		if err != nil {
+			setupLog.Error(err, "unable to set up cert rotation")
+			os.Exit(1)
+		}
+	} else {
+		close(certsReady)
 	}
 
 	setupProbeEndpoints(mgr, certsReady)
@@ -125,7 +134,7 @@ func parseFlags() {
 		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
 	flag.StringVar(&leaderElectionId, "leader-election-id", "controller-leader-election-helper",
 		"Leader election id determines the name of the configmap that leader election will use for holding the leader lock.")
-	flag.BoolVar(&novalidation, "novalidation", false, "Disables validating webhook")
+	flag.BoolVar(&noWebhooks, "no-webhooks", false, "Disables webhooks")
 	flag.BoolVar(&debugLogs, "debug-logs", false, "Shows verbose logs.")
 	flag.BoolVar(&testLog, "enable-test-log", false, "Enables test log.")
 	flag.BoolVar(&internalCert, "enable-internal-cert-management", false, "Enables internal cert management. See the user guide for more information.")
@@ -139,6 +148,7 @@ func parseFlags() {
 	flag.BoolVar(&restartOnSecretRefresh, "cert-restart-on-secret-refresh", false, "Kills the process when secrets are refreshed so that the pod can be restarted (secrets take up to 60s to be updated by running pods)")
 	flag.Var(&managedNamespaceLabels, "managed-namespace-label", "A regex indicating the labels on namespaces that are managed by HNC. These labels may only be set via the HierarchyConfiguration object. All regexes are implictly wrapped by \"^...$\". This argument can be specified multiple times. See the user guide for more information.")
 	flag.Var(&managedNamespaceAnnots, "managed-namespace-annotation", "A regex indicating the annotations on namespaces that are managed by HNC. These annotations may only be set via the HierarchyConfiguration object. All regexes are implictly wrapped by \"^...$\". This argument can be specified multiple times. See the user guide for more information.")
+	flag.BoolVar(&webhooksOnly, "webhooks-only", false, "Disables the controllers so HNC can be run in HA webhook mode")
 	flag.Parse()
 
 	// Assign the array args to the configuration variables after the args are parsed.
@@ -148,6 +158,12 @@ func parseFlags() {
 		setupLog.Error(err, "Illegal flag values")
 		os.Exit(1)
 	}
+
+	// Basic legality checks
+	if webhooksOnly && noWebhooks {
+		setupLog.Info("Cannot set both --webhooks-only and --no-webhooks")
+		os.Exit(1)
+	}
 }
 
 // enableMetrics returns a function to call from main() to export any remaining metrics when main()
@@ -252,6 +268,10 @@ func setupProbeEndpoints(mgr ctrl.Manager, certsReady chan struct{}) {
 			return errors.New("HNC internal certs are not yet ready")
 		}
 	}
+	// If we're not running the webhooks, no point checking to see if they're up.
+	if noWebhooks {
+		checker = healthz.Ping
+	}
 	if err := mgr.AddHealthzCheck("healthz", checker); err != nil {
 		setupLog.Error(err, "unable to set up health check")
 		os.Exit(1)
@@ -278,14 +298,14 @@ func startControllers(mgr ctrl.Manager, certsReady chan struct{}) {
 	f := forest.NewForest()
 
 	// Create all validating and mutating admission controllers.
-	if !novalidation {
-		setupLog.Info("Registering validating webhook (won't work when running locally; use --novalidation)")
+	if !noWebhooks {
+		setupLog.Info("Registering validating webhook (won't work when running locally; use --no-webhooks)")
 		setup.CreateWebhooks(mgr, f)
 	}
 
 	// Create all reconciling controllers
 	setupLog.Info("Creating controllers", "maxReconciles", maxReconciles)
-	if err := setup.CreateReconcilers(mgr, f, maxReconciles, false); err != nil {
+	if err := setup.CreateReconcilers(mgr, f, maxReconciles, webhooksOnly, false); err != nil {
 		setupLog.Error(err, "cannot create controllers")
 		os.Exit(1)
 	}

internal/anchor/reconciler.go

@@ -51,6 +51,9 @@ type Reconciler struct {
 	// https://book-v1.book.kubebuilder.io/beyond_basics/controller_watches.html) that is used to
 	// enqueue additional objects that need updating.
 	Affected chan event.GenericEvent
+
+	// ReadOnly disables writebacks
+	ReadOnly bool
 }
 
 // Reconcile sets up some basic variables and then calls the business logic. It currently
@@ -339,6 +342,10 @@ func (r *Reconciler) getInstance(ctx context.Context, pnm, nm string) (*api.Subn
 }
 
 func (r *Reconciler) writeInstance(ctx context.Context, log logr.Logger, inst *api.SubnamespaceAnchor) error {
+	if r.ReadOnly {
+		return nil
+	}
+
 	if inst.CreationTimestamp.IsZero() {
 		if err := r.Create(ctx, inst); err != nil {
 			log.Error(err, "while creating on apiserver")
@@ -356,6 +363,10 @@ func (r *Reconciler) writeInstance(ctx context.Context, log logr.Logger, inst *a
 // deleteInstance deletes the anchor instance. Note: Make sure there's no
 // finalizers on the instance before calling this function.
 func (r *Reconciler) deleteInstance(ctx context.Context, inst *api.SubnamespaceAnchor) error {
+	if r.ReadOnly {
+		return nil
+	}
+
 	if err := r.Delete(ctx, inst); err != nil {
 		return fmt.Errorf("while deleting on apiserver: %w", err)
 	}
@@ -378,6 +389,10 @@ func (r *Reconciler) getNamespace(ctx context.Context, nm string) (*corev1.Names
 }
 
 func (r *Reconciler) writeNamespace(ctx context.Context, log logr.Logger, nm, pnm string) error {
+	if r.ReadOnly {
+		return nil
+	}
+
 	inst := &corev1.Namespace{}
 	inst.ObjectMeta.Name = nm
 	metadata.SetAnnotation(inst, api.SubnamespaceOf, pnm)
@@ -395,6 +410,10 @@ func (r *Reconciler) writeNamespace(ctx context.Context, log logr.Logger, nm, pn
 }
 
 func (r *Reconciler) deleteNamespace(ctx context.Context, log logr.Logger, inst *corev1.Namespace) error {
+	if r.ReadOnly {
+		return nil
+	}
+
 	if err := r.Delete(ctx, inst); err != nil {
 		log.Error(err, "While deleting subnamespace")
 		return err

internal/hierarchyconfig/reconciler.go

@@ -78,6 +78,8 @@ type Reconciler struct {
 	// These are interfaces to the other reconcilers
 	AnchorReconciler    AnchorReconcilerType
 	HNCConfigReconciler HNCConfigReconcilerType
+
+	ReadOnly bool
 }
 
 type AnchorReconcilerType interface {
@@ -704,6 +706,10 @@ func (r *Reconciler) writeHierarchy(ctx context.Context, log logr.Logger, orig,
 	if reflect.DeepEqual(orig, inst) {
 		return false, nil
 	}
+	if r.ReadOnly {
+		// We *wanted* to change something, so assume that we need to re-reconcile other objects as well
+		return true, nil
+	}
 	exists := !inst.CreationTimestamp.IsZero()
 	if !exists && isDeletingNS {
 		log.Info("Will not create hierarchyconfiguration since namespace is being deleted")
@@ -732,6 +738,10 @@ func (r *Reconciler) writeNamespace(ctx context.Context, log logr.Logger, orig,
 	if reflect.DeepEqual(orig, inst) {
 		return false, nil
 	}
+	if r.ReadOnly {
+		// We *wanted* to change something, so assume that we need to re-reconcile other objects as well
+		return true, nil
+	}
 
 	// NB: HCR can't create namespaces, that's only in anchor reconciler
 	stats.WriteNamespace()

internal/hncconfig/reconciler.go

@@ -53,6 +53,9 @@ type Reconciler struct {
 	// not use it.
 	HierarchyConfigUpdates chan event.GenericEvent
 
+	// ReadOnly disables writebacks
+	ReadOnly bool
+
 	// activeGVKMode contains GRs that are configured in the Spec and their mapping
 	// GVKs and configured modes.
 	activeGVKMode gr2gvkMode
@@ -233,6 +236,10 @@ func (r *Reconciler) validateSingleton(inst *api.HNCConfiguration) {
 // We will write the singleton to apiserver even it is not changed because we assume this
 // reconciler is called very infrequently and is not performance critical.
 func (r *Reconciler) writeSingleton(ctx context.Context, inst *api.HNCConfiguration) error {
+	if r.ReadOnly {
+		return nil
+	}
+
 	if inst.CreationTimestamp.IsZero() {
 		// No point creating it if the CRD's being deleted
 		if isDeleted, err := crd.IsDeletingCRD(ctx, api.HNCConfigSingletons); isDeleted || err != nil {

internal/integtest/setup.go

@@ -102,7 +102,7 @@ func HNCBeforeSuite() {
 	Expect(err).ToNot(HaveOccurred())
 
 	By("creating reconcilers")
-	err = setup.CreateReconcilers(k8sManager, forest.NewForest(), 100, true)
+	err = setup.CreateReconcilers(k8sManager, forest.NewForest(), 100, false, true)
 	Expect(err).ToNot(HaveOccurred())
 
 	By("Creating clients")

internal/setup/reconcilers.go

@@ -17,7 +17,7 @@ import (
 // CreateReconcilers creates all reconcilers.
 //
 // This function is called both from main.go as well as from the integ tests.
-func CreateReconcilers(mgr ctrl.Manager, f *forest.Forest, maxReconciles int, useFakeClient bool) error {
+func CreateReconcilers(mgr ctrl.Manager, f *forest.Forest, maxReconciles int, readOnly, useFakeClient bool) error {
 	if err := crd.Setup(mgr, useFakeClient); err != nil {
 		return err
 	}
@@ -31,6 +31,7 @@ func CreateReconcilers(mgr ctrl.Manager, f *forest.Forest, maxReconciles int, us
 		Log:      ctrl.Log.WithName("anchor").WithName("reconcile"),
 		Forest:   f,
 		Affected: anchorChan,
+		ReadOnly: readOnly,
 	}
 	if err := ar.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("cannot create anchor reconciler: %s", err.Error())
@@ -50,6 +51,7 @@ func CreateReconcilers(mgr ctrl.Manager, f *forest.Forest, maxReconciles int, us
 		Forest:                 f,
 		Trigger:                make(chan event.GenericEvent),
 		HierarchyConfigUpdates: hcChan,
+		ReadOnly:               readOnly,
 	}
 	if err := hnccfgr.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("cannot create Config reconciler: %s", err.Error())
@@ -63,6 +65,7 @@ func CreateReconcilers(mgr ctrl.Manager, f *forest.Forest, maxReconciles int, us
 		AnchorReconciler:    ar,
 		HNCConfigReconciler: hnccfgr,
 		Affected:            hcChan,
+		ReadOnly:            readOnly,
 	}
 	if err := hcr.SetupWithManager(mgr, maxReconciles); err != nil {
 		return fmt.Errorf("cannot create Hierarchy reconciler: %s", err.Error())

internal/setup/webhooks.go

@@ -30,15 +30,9 @@ const (
 // DNSName is <service name>.<namespace>.svc
 var dnsName = fmt.Sprintf("%s.%s.svc", serviceName, secretNamespace)
 
-// CreateCertsIfNeeded creates all certs for webhooks. This function is called from main.go.
-func CreateCertsIfNeeded(mgr ctrl.Manager, novalidation, internalCert, restartOnSecretRefresh bool) (chan struct{}, error) {
-	setupFinished := make(chan struct{})
-	if novalidation || !internalCert {
-		close(setupFinished)
-		return setupFinished, nil
-	}
-
-	return setupFinished, cert.AddRotator(mgr, &cert.CertRotator{
+// ManageCerts creates all certs for webhooks. This function is called from main.go.
+func ManageCerts(mgr ctrl.Manager, setupFinished chan struct{}, restartOnSecretRefresh bool) error {
+	return cert.AddRotator(mgr, &cert.CertRotator{
 		SecretKey: types.NamespacedName{
 			Namespace: secretNamespace,
 			Name:      secretName,