adding vn-agent healthcheck & ability to boot from kconfig

Signed-off-by: Chris Hein <me@chrishein.com>

Author: christopherhein

virtualcluster/cmd/vn-agent/app/options/options.go

@@ -51,6 +51,9 @@ type ServerOption struct {
 	// Port is the vn-agent server listening on.
 	Port uint
 
+	// Kubeconfig is the supercluster Kubeconfig to connect to
+	Kubeconfig string
+
 	// FeatureGates enabled by the user.
 	FeatureGates map[string]bool
 }
@@ -84,6 +87,7 @@ func (o *Options) Flags() cliflag.NamedFlagSets {
 	serverFS.StringVar(&o.CertDirectory, "cert-dir", o.CertDirectory, "CertDirectory is the directory where the TLS certs are located")
 	serverFS.StringVar(&o.TLSCertFile, "tls-cert-file", o.TLSCertFile, "TLSCertFile is the file containing x509 Certificate for HTTPS")
 	serverFS.StringVar(&o.TLSPrivateKeyFile, "tls-private-key-file", o.TLSPrivateKeyFile, "TLSPrivateKeyFile is the file containing x509 private key matching tlsCertFile")
+	serverFS.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
 	serverFS.UintVar(&o.Port, "port", 10550, "Port is the server listening on")
 	serverFS.Var(cliflag.NewMapStringBool(&o.ServerOption.FeatureGates), "feature-gates", "A set of key=value pairs that describe featuregate gates for various features.")
 

virtualcluster/cmd/vn-agent/app/server.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/component-base/cli/globalflag"
 	"k8s.io/component-base/term"
 	"k8s.io/klog"
+	"k8s.io/kubernetes/pkg/healthz"
 
 	"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/cmd/vn-agent/app/options"
 	utilflag "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/util/flag"
@@ -90,7 +91,7 @@ func NewVnAgentCommand(stopChan <-chan struct{}) *cobra.Command {
 
 // Run start the vn-agent server.
 func Run(c *config.Config, serverOption *options.ServerOption, stopCh <-chan struct{}) error {
-	handler, err := server.NewServer(c)
+	handler, err := server.NewServer(c, serverOption)
 	if err != nil {
 		return errors.Wrapf(err, "create server")
 	}
@@ -131,6 +132,14 @@ func Run(c *config.Config, serverOption *options.ServerOption, stopCh <-chan str
 		errCh <- err
 	}()
 
+	go func() {
+		// start a health http server.
+		mux := http.NewServeMux()
+		healthz.InstallHandler(mux)
+		klog.Fatal(http.ListenAndServe(":8080", mux))
+		errCh <- err
+	}()
+
 	select {
 	case <-stopCh:
 		klog.Infof("closing server...")

virtualcluster/go.sum

@@ -1082,6 +1082,7 @@ k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhD
 k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
 k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
 k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
+k8s.io/kubernetes v0.20.2 h1:KxQftTwyg3DRlwKDcI6wla6eLygmMJ21FXlE4o7QcvU=
 k8s.io/kubernetes v0.20.2/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ=

virtualcluster/pkg/controller/controllers/suite_test.go

@@ -44,8 +44,8 @@ var cfg *rest.Config
 var cli client.Client
 var testEnv *envtest.Environment
 
-const timeout = time.Second * 10
-const longTimeout = time.Second * 20
+const timeout = time.Second * 20
+const longTimeout = time.Second * 60
 const interval = time.Millisecond * 250
 
 func TestAPIs(t *testing.T) {

virtualcluster/pkg/vn-agent/server/server.go

@@ -18,14 +18,17 @@ package server
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"net/http"
 	"net/url"
 
 	"github.com/emicklei/go-restful"
 	"github.com/pkg/errors"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 	certutil "k8s.io/client-go/util/cert"
 
+	"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/cmd/vn-agent/app/options"
 	"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/vn-agent/config"
 )
 
@@ -44,7 +47,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 }
 
 // NewServer initializes and configures a vn-agent.Server object to handle HTTP requests.
-func NewServer(cfg *config.Config) (*Server, error) {
+func NewServer(cfg *config.Config, serverOption *options.ServerOption) (*Server, error) {
 	u, err := url.Parse(cfg.KubeletServerHost)
 	if err != nil {
 		return nil, errors.Wrap(err, "parse kubelet server url")
@@ -66,17 +69,27 @@ func NewServer(cfg *config.Config) (*Server, error) {
 			},
 		}
 	} else {
-		restConfig, err := rest.InClusterConfig()
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to get in cluster config")
+		var restConfig *rest.Config
+		var caCrtPool *x509.CertPool
+		if len(serverOption.Kubeconfig) == 0 {
+			restConfig, err = rest.InClusterConfig()
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to get in cluster config")
+			}
+			caCrtPool, err = certutil.NewPool(restConfig.TLSClientConfig.CAFile)
+		} else {
+			// This creates a client, first loading any specified kubeconfig\
+			restConfig, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+				&clientcmd.ClientConfigLoadingRules{ExplicitPath: serverOption.Kubeconfig},
+				&clientcmd.ConfigOverrides{}).ClientConfig()
+			caCrtPool, err = certutil.NewPoolFromBytes(restConfig.TLSClientConfig.CAData)
 		}
 		server.restConfig = restConfig
 		superHttpsUrl, err := url.Parse(restConfig.Host)
 		if err != nil {
 			return nil, errors.Wrapf(err, "unable to parse apiserver address")
 		}
 		server.superAPIServerAddress = superHttpsUrl
-		caCrtPool, err := certutil.NewPool(restConfig.TLSClientConfig.CAFile)
 		if err != nil {
 			return nil, errors.Wrapf(err, "unable to parse ca file")
 		}

virtualcluster/pkg/vn-agent/server/test/go.sum

@@ -548,6 +548,7 @@ github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmK
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.10 h1:6q5mVkdH/vYmqngx7kZQTjJ5HRsx+ImorDIEQ+beJgc=
 github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=

virtualcluster/pkg/vn-agent/server/test/server_test.go

@@ -61,6 +61,7 @@ import (
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/utils/pointer"
 
+	"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/cmd/vn-agent/app/options"
 	"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/vn-agent/config"
 	"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/vn-agent/server"
 	"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/vn-agent/testcerts"
@@ -124,7 +125,7 @@ func newServerTestWithDebug(enableDebugging bool, streamingServer streaming.Serv
 	server, err := server.NewServer(&config.Config{
 		KubeletClientCert: &kubeletClientCert,
 		KubeletServerHost: fv.kubeletServer.testHTTPServer.URL,
-	})
+	}, &options.ServerOption{})
 	if err != nil {
 		panic(errors.Wrap(err, "new server"))
 	}