Merge pull request #133 from JoshVanL/dev-cluster-deploy-improvments

Dev cluster deploy improvments

Author: jetstack-bot

.dockerignore

@@ -1,2 +1,2 @@
 *
-!/bin/kube-oidc-proxy
+!/bin/kube-oidc-proxy-linux

Dockerfile

@@ -4,6 +4,6 @@ LABEL description="OIDC reverse proxy authenticator based on Kubernetes"
 
 RUN apk --no-cache --update add ca-certificates
 
-COPY ./bin/kube-oidc-proxy /usr/bin/kube-oidc-proxy
+COPY ./bin/kube-oidc-proxy-linux /usr/bin/kube-oidc-proxy
 
 CMD ["/usr/bin/kube-oidc-proxy"]

Makefile

@@ -3,6 +3,7 @@ BINDIR    ?= $(CURDIR)/bin
 HACK_DIR  ?= hack
 PATH      := $(BINDIR):$(PATH)
 ARTIFACTS ?= artifacts
+ARCH      ?= amd64
 
 SHELL = /bin/bash -o pipefail
 
@@ -91,25 +92,26 @@ test: generate verify ## run all go tests
 	go test -v -bench $$(go list ./pkg/... ./cmd/... | grep -v pkg/e2e) | tee $(ARTIFACTS)/go-test.stdout
 	cat $(ARTIFACTS)/go-test.stdout | go run github.com/jstemmer/go-junit-report > $(ARTIFACTS)/junit-go-test.xml
 
-e2e: ## run end to end tests
+e2e: depend ## run end to end tests
 	mkdir -p $(ARTIFACTS)
 	KUBE_OIDC_PROXY_ROOT_PATH="$$(pwd)" go test -timeout 30m -v --count=1 ./test/e2e/suite/.
 
 build: generate ## build kube-oidc-proxy
 	CGO_ENABLED=0 go build -ldflags '-w $(shell hack/version-ldflags.sh)' -o ./bin/kube-oidc-proxy ./cmd/.
 
 docker_build: generate test build ## build docker image
+	GOARCH=$(ARCH) GOOS=linux CGO_ENABLED=0 go build -ldflags '-w $(shell hack/version-ldflags.sh)' -o ./bin/kube-oidc-proxy-linux  ./cmd/.
 	docker build -t kube-oidc-proxy .
 
 all: test build ## runs tests, build
 
 image: all docker_build ## runs tests, build and docker build
 
-dev_cluster_create: ## create dev cluster for development testing
+dev_cluster_create: depend ## create dev cluster for development testing
 	KUBE_OIDC_PROXY_ROOT_PATH="$$(pwd)" go run -v ./test/environment/dev create
 
-dev_cluster_deploy: ## deploy into dev cluster
+dev_cluster_deploy: depend ## deploy into dev cluster
 	KUBE_OIDC_PROXY_ROOT_PATH="$$(pwd)" go run -v ./test/environment/dev deploy
 
-dev_cluster_destroy: ## destroy dev cluster
+dev_cluster_destroy: depend ## destroy dev cluster
 	KUBE_OIDC_PROXY_ROOT_PATH="$$(pwd)" go run -v ./test/environment/dev destroy

README.md

@@ -132,3 +132,7 @@ users:
 
 ## Development
 *NOTE*: building kube-oidc-proxy requires Go version 1.12 or higher.
+
+To help with development, there is a suite of tools you can use to deploy a
+functioning proxy from source locally. You can read more
+[here](./docs/tasks/development-testing.md).

docs/tasks/development-testing.md

@@ -0,0 +1,42 @@
+# Development Testing
+
+In order to help development for the proxy, there are a few tools in place for
+quick testing.
+
+# Creating a Cluster
+
+Use `make dev_cluster_create` to spin up a kind cluster locally. This will also
+build the proxy and other tooling from source, build their images, and load them
+onto each node.
+
+# Deploying the Proxy
+
+This will build the proxy and other tooling from source,build the images, and
+load them onto each node. This will then deploy the proxy alongside a fake OIDC
+issuer so that the proxy is fully functional. The proxy will then be reachable
+from a node port service in the cluster.
+
+
+```bash
+make dev_cluster_deploy
+```
+
+This command will output a signed OIDC token that is valid for the proxy. You
+can then make calls to the proxy, like the following:
+
+```bash
+curl -k https://172.17.0.2:30226 -H 'Authorization: bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.ewoJImlzcyI6Imh0dHBzOi8vb2lkYy1pc3N1ZXItZTJlLmt1YmUtb2lkYy1wcm94eS1lMmUtNmhiNGcuc3ZjLmNsdXN0ZXIubG9jYWw6NjQ0MyIsCgkiYXVkIjpbImt1YmUtb2lkYy1wcm94eS1lMmUtY2xpZW50LWlkIiwiYXVkLTIiXSwKCSJlbWFpbCI6InVzZXJAZXhhbXBsZS5jb20iLAoJImdyb3VwcyI6WyJncm91cC0xIiwiZ3JvdXAtMiJdLAoJImV4cCI6MTU4MjU1NTYzMQoJfQ.qWCM5zUHGslmwbgyZnMjhVeCLJd3R3c7xjtatjT_pv1VY-PpJ8IGBsbcCpur1fAm2CAbr0juM3yzwV1S3TUjhNhE8Wo6rxjA2Flnmwj7Nn2Got6T_cMFHQ_3A6YC72qkMwH-7SvXFB-C5Bk96vi9-clrxJ_b1XjfMPViZEVCJphh9HVzrZ5DPOAR0PDl-qnVys_CRkF0NEwEvAZL5SFumBqjtLBI9XUlWbB6VTljPOExL1zkv8NevZF8DxVsYFaW9HOYH8vNgC07kj_oUVkmAjP-2tVngcBKka0IBmuz2r-RfWNy9VJ-yb19AbtJNw6fjASy7O6VifuH4ZpjP5JSIg'
+```
+
+You are also able to deploy a server that the proxy connects to. This is useful
+for checking the headers and request body sent to the target server by the
+proxy which are present in the server logs. To enable this, set the following
+environment variable:
+
+```bash
+KUBE_OIDC_PROXY_FAKE_APISERVER=true make dev_cluster_deploy
+```
+
+# Delete the cluster
+
+To delete the test kind cluster, use `make dev_cluster_destroy`.

test/e2e/framework/helper/deploy.go

@@ -209,7 +209,7 @@ func (h *Helper) DeployIssuer(ns string) (*util.KeyBundle, *url.URL, error) {
 	return bundle, appURL, nil
 }
 
-func (h *Helper) DeployFakeAPIServer(ns string) (*util.KeyBundle, *url.URL, error) {
+func (h *Helper) DeployFakeAPIServer(ns string) ([]corev1.Volume, *url.URL, error) {
 	cnt := corev1.Container{
 		Name:            FakeAPIServerName,
 		Image:           FakeAPIServerName,
@@ -239,7 +239,31 @@ func (h *Helper) DeployFakeAPIServer(ns string) (*util.KeyBundle, *url.URL, erro
 		return nil, nil, err
 	}
 
-	return bundle, appURL, nil
+	sec, err := h.KubeClient.CoreV1().Secrets(ns).Create(&corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "fake-apiserver-ca-",
+			Namespace:    ns,
+		},
+		Data: map[string][]byte{
+			"ca.pem": bundle.CertBytes,
+		},
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	extraVolumes := []corev1.Volume{
+		{
+			Name: "fake-apiserver",
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: sec.Name,
+				},
+			},
+		},
+	}
+
+	return extraVolumes, appURL, nil
 }
 
 func (h *Helper) deployApp(ns, name string, serviceType corev1.ServiceType, container corev1.Container, volumes ...corev1.Volume) (*util.KeyBundle, *url.URL, error) {

test/e2e/suite/cases/headers/headers.go

@@ -4,16 +4,12 @@ package headers
 import (
 	"fmt"
 	"net/http"
-	"net/url"
 	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"github.com/jetstack/kube-oidc-proxy/test/e2e/framework"
 	testutil "github.com/jetstack/kube-oidc-proxy/test/util"
 )
@@ -28,10 +24,11 @@ var _ = framework.CasesDescribe("Headers", func() {
 	})
 
 	It("should not respond with any extra headers if none are set on the proxy", func() {
-		fakeAPIServerURL, extraVolumes := deployFakeAPIServer(f)
+		extraOIDCVolumes, fakeAPIServerURL, err := f.Helper().DeployFakeAPIServer(f.Namespace.Name)
+		Expect(err).NotTo(HaveOccurred())
 
 		By("Redeploying proxy to send traffic to fake API server")
-		f.DeployProxyWith(extraVolumes, fmt.Sprintf("--server=%s", fakeAPIServerURL), "--certificate-authority=/fake-apiserver/ca.pem")
+		f.DeployProxyWith(extraOIDCVolumes, fmt.Sprintf("--server=%s", fakeAPIServerURL), "--certificate-authority=/fake-apiserver/ca.pem")
 
 		resp := sendRequestToProxy(f)
 
@@ -44,10 +41,12 @@ var _ = framework.CasesDescribe("Headers", func() {
 	})
 
 	It("should respond with remote address and custom extra headers when they are set", func() {
-		fakeAPIServerURL, extraVolumes := deployFakeAPIServer(f)
+		By("Deploying fake API Server")
+		extraOIDCVolumes, fakeAPIServerURL, err := f.Helper().DeployFakeAPIServer(f.Namespace.Name)
+		Expect(err).NotTo(HaveOccurred())
 
 		By("Redeploying proxy to send traffic to fake API server with extra headers set")
-		f.DeployProxyWith(extraVolumes, fmt.Sprintf("--server=%s", fakeAPIServerURL), "--certificate-authority=/fake-apiserver/ca.pem",
+		f.DeployProxyWith(extraOIDCVolumes, fmt.Sprintf("--server=%s", fakeAPIServerURL), "--certificate-authority=/fake-apiserver/ca.pem",
 			"--extra-user-header-client-ip", "--extra-user-headers=key1=foo,key2=foo,key1=bar")
 
 		resp := sendRequestToProxy(f)
@@ -86,36 +85,6 @@ var _ = framework.CasesDescribe("Headers", func() {
 	})
 })
 
-func deployFakeAPIServer(f *framework.Framework) (*url.URL, []corev1.Volume) {
-	By("Deploying fake API Server")
-	fAPIServerBundle, fakeAPIServerURL, err := f.Helper().DeployFakeAPIServer(f.Namespace.Name)
-	Expect(err).NotTo(HaveOccurred())
-
-	sec, err := f.KubeClientSet.CoreV1().Secrets(f.Namespace.Name).Create(&corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: "fake-apiserver-ca-",
-			Namespace:    f.Namespace.Name,
-		},
-		Data: map[string][]byte{
-			"ca.pem": fAPIServerBundle.CertBytes,
-		},
-	})
-	Expect(err).NotTo(HaveOccurred())
-
-	extraVolumes := []corev1.Volume{
-		{
-			Name: "fake-apiserver",
-			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{
-					SecretName: sec.Name,
-				},
-			},
-		},
-	}
-
-	return fakeAPIServerURL, extraVolumes
-}
-
 func sendRequestToProxy(f *framework.Framework) *http.Response {
 	By("Building request to proxy")
 	tokenPayload := f.Helper().NewTokenPayload(

test/e2e/suite/suite.go

@@ -18,9 +18,13 @@ var (
 
 var _ = SynchronizedBeforeSuite(func() []byte {
 	var err error
-	env, err = environment.Create(1, 0)
+	env, err = environment.New(1, 0)
 	if err != nil {
-		log.Fatalf("Error provisioning environment: %v", err)
+		log.Fatalf("Error provisioning environment: %s", err)
+	}
+
+	if err := env.Create(); err != nil {
+		log.Fatalf("Error creating environment: %s", err)
 	}
 
 	cfg.KubeConfigPath = env.KubeConfigPath()
@@ -29,7 +33,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	cfg.Environment = env
 
 	if err := framework.DefaultConfig.Validate(); err != nil {
-		log.Fatalf("Invalid test config: %v", err)
+		log.Fatalf("Invalid test config: %s", err)
 	}
 
 	return nil

test/environment/dev/dev.go

@@ -3,8 +3,10 @@ package main
 
 import (
 	"fmt"
+	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
@@ -20,6 +22,8 @@ import (
 
 const (
 	clientID = "kube-oidc-proxy-e2e-client-id"
+
+	EnvFakeAPIServer = "KUBE_OIDC_PROXY_FAKE_APISERVER"
 )
 
 func main() {
@@ -44,23 +48,28 @@ func main() {
 }
 
 func create() {
-	env, err := environment.Create(1, 1)
+	env, err := environment.New(1, 1)
 	errExit(err)
 
+	errExit(env.Create())
+
 	fmt.Printf("> dev environment created.\n")
 	fmt.Printf("export KUBECONFIG=%s\n", env.KubeConfigPath())
 }
 
 func deploy() {
-	k := new(kind.Kind)
-	kubeconfig := k.KubeConfigPath()
-	rootPath, err := environment.RootPath()
+	env, err := environment.New(1, 1)
 	errExit(err)
 
+	fmt.Printf("> reloading all images\n")
+	errExit(env.Kind().LoadAllImages())
+
+	kubeconfig := env.KubeConfigPath()
+
 	cfg := &config.Config{
 		KubeConfigPath: kubeconfig,
-		RepoRoot:       rootPath,
-		Kubectl:        filepath.Join(rootPath, "bin", "kubectl"),
+		RepoRoot:       env.RootPath(),
+		Kubectl:        filepath.Join(env.RootPath(), "bin", "kubectl"),
 	}
 
 	err = cfg.Validate()
@@ -93,9 +102,24 @@ func deploy() {
 
 	fmt.Printf("> deployed issuer at url %s\n", issuerURL)
 
-	_, proxyURL, err := helper.DeployProxy(ns, issuerURL,
-		"kube-oidc-proxy-e2e-client-id", issuerKeyBundle, nil)
-	errExit(err)
+	var proxyURL *url.URL
+	if e := os.Getenv(EnvFakeAPIServer); strings.ToLower(e) == "true" {
+		extraOIDCVolume, fURL, err := helper.DeployFakeAPIServer(ns.Name)
+		errExit(err)
+
+		fmt.Printf("> deployed fake API server at url %s\n", fURL)
+
+		_, proxyURL, err = helper.DeployProxy(ns, issuerURL,
+			"kube-oidc-proxy-e2e-client-id", issuerKeyBundle, extraOIDCVolume,
+			fmt.Sprintf("--server=%s", fURL), "--certificate-authority=/fake-apiserver/ca.pem")
+		errExit(err)
+
+	} else {
+		_, proxyURL, err = helper.DeployProxy(ns, issuerURL,
+			"kube-oidc-proxy-e2e-client-id", issuerKeyBundle, nil)
+		errExit(err)
+	}
+
 	fmt.Printf("> deployed proxy at url %s\n", proxyURL)
 
 	tokenPayload := helper.NewTokenPayload(issuerURL, clientID, time.Now().Add(time.Hour*48))