Skip to content

Commit 06ab683

Browse files
kvapskvaps
committed
add konnectivity component
1 parent d3149d5 commit 06ab683

14 files changed

+575
-7
lines changed

‎deploy/helm/kubernetes/manifests/konnectivity-agent-deployment.yaml‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
{{- $fullName := include "kubernetes.fullname" . -}}
2+
apiVersion: apps/v1
3+
kind: Deployment
4+
metadata:
5+
labels:
6+
addonmanager.kubernetes.io/mode: Reconcile
7+
k8s-app: konnectivity-agent
8+
{{- with .Values.konnectivityAgent.labels }}
9+
{{- toYaml . | nindent 4 }}
10+
{{- end }}
11+
{{- with .Values.konnectivityAgent.annotations }}
12+
annotations:
13+
{{- toYaml . | nindent 4 }}
14+
{{- end }}
15+
namespace: kube-system
16+
name: konnectivity-agent
17+
spec:
18+
replicas: {{ .Values.konnectivityAgent.replicaCount }}
19+
selector:
20+
matchLabels:
21+
k8s-app: konnectivity-agent
22+
template:
23+
metadata:
24+
labels:
25+
k8s-app: konnectivity-agent
26+
{{- with .Values.konnectivityAgent.podLabels }}
27+
{{- toYaml . | nindent 8 }}
28+
{{- end }}
29+
{{- with .Values.konnectivityAgent.podAnnotations }}
30+
annotations:
31+
{{- toYaml . | nindent 8 }}
32+
{{- end }}
33+
spec:
34+
{{- with .Values.konnectivityAgent.nodeSelector }}
35+
nodeSelector:
36+
{{- toYaml . | nindent 8 }}
37+
{{- end }}
38+
hostNetwork: {{ .Values.konnectivityAgent.hostNetwork }}
39+
priorityClassName: system-cluster-critical
40+
tolerations:
41+
- key: "CriticalAddonsOnly"
42+
operator: "Exists"
43+
{{- with .Values.konnectivityAgent.tolerations }}
44+
{{- toYaml . | nindent 6 }}
45+
{{- end }}
46+
containers:
47+
- image: {{ .Values.konnectivityAgent.image.repository }}:{{ .Values.konnectivityAgent.image.tag }}
48+
name: konnectivity-agent
49+
command: ["/proxy-agent"]
50+
args:
51+
- --logtostderr=true
52+
- --ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
53+
- --service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token
54+
55+
{{- if not (hasKey .Values.konnectivityAgent.extraArgs "proxy-server-host") }}
56+
{{- if .Values.konnectivityServer.service.loadBalancerIP }}
57+
- --proxy-server-host={{ .Values.konnectivityServer.service.loadBalancerIP }}
58+
{{- else }}
59+
{{- fail ".konnectivityAgent.extraArgs.proxy-server-host must be specified!" }}
60+
{{- end }}
61+
{{- end }}
62+
63+
{{- if not (hasKey .Values.konnectivityAgent.extraArgs "roxy-server-port") }}
64+
{{- if eq .Values.konnectivityServer.service.type "LoadBalancer" }}
65+
- --proxy-server-port={{ .Values.konnectivityServer.service.ports.agent }}
66+
{{- else if .Values.konnectivityServer.service.NodePort }}
67+
- --proxy-server-port={{ .Values.konnectivityServer.service.nodePorts.agent }}
68+
{{- else }}
69+
{{- fail ".konnectivityAgent.extraArgs.proxy-server-port must be specified!" }}
70+
{{- end }}
71+
{{- end }}
72+
73+
{{- range $key, $value := .Values.konnectivityAgent.extraArgs }}
74+
- --{{ $key }}={{ $value }}
75+
{{- end }}
76+
volumeMounts:
77+
- mountPath: /var/run/secrets/tokens
78+
name: konnectivity-agent-token
79+
{{- with .Values.konnectivityAgent.extraVolumeMounts }}
80+
{{- toYaml . | nindent 8 }}
81+
{{- end }}
82+
livenessProbe:
83+
httpGet:
84+
port: 8093
85+
path: /healthz
86+
initialDelaySeconds: 15
87+
timeoutSeconds: 15
88+
{{- with .Values.konnectivityAgent.sidecars }}
89+
{{- toYaml . | nindent 6 }}
90+
{{- end }}
91+
serviceAccountName: konnectivity-agent
92+
volumes:
93+
- name: konnectivity-agent-token
94+
projected:
95+
sources:
96+
- serviceAccountToken:
97+
path: konnectivity-agent-token
98+
audience: system:konnectivity-server
99+
{{- with .Values.konnectivityAgent.extraVolumes }}
100+
{{- toYaml . | nindent 6 }}
101+
{{- end }}

‎deploy/helm/kubernetes/manifests/konnectivity-agent-rbac.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: konnectivity-agent
5+
namespace: kube-system
6+
labels:
7+
kubernetes.io/cluster-service: "true"
8+
addonmanager.kubernetes.io/mode: Reconcile

‎deploy/helm/kubernetes/manifests/konnectivity-server-rbac.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRoleBinding
3+
metadata:
4+
name: system:konnectivity-server
5+
labels:
6+
kubernetes.io/cluster-service: "true"
7+
addonmanager.kubernetes.io/mode: Reconcile
8+
roleRef:
9+
apiGroup: rbac.authorization.k8s.io
10+
kind: ClusterRole
11+
name: system:auth-delegator
12+
subjects:
13+
- apiGroup: rbac.authorization.k8s.io
14+
kind: User
15+
name: system:konnectivity-server

‎deploy/helm/kubernetes/scripts/configure-cluster.sh‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,25 +42,36 @@ fi
4242
# generate cluster-admin kubeconfig
4343
rm -f /etc/kubernetes/admin.conf
4444
kubeadm init phase kubeconfig admin --config kubeadmcfg.yaml
45-
kubectl --kubeconfig=/etc/kubernetes/admin.conf config set clusters.kubernetes.server "https://${FULL_NAME}-apiserver:6443"
45+
kubectl --kubeconfig=/etc/kubernetes/admin.conf config set-cluster kubernetes --server "https://${FULL_NAME}-apiserver:6443"
4646
kubectl create secret generic "${FULL_NAME}-admin-conf" --from-file=/etc/kubernetes/admin.conf --dry-run=client -o yaml | kubectl apply -f -
4747

4848
{{- if .Values.controllerManager.enabled }}{{"\n"}}
4949
# generate controller-manager kubeconfig
5050
rm -f /etc/kubernetes/controller-manager.conf
5151
kubeadm init phase kubeconfig controller-manager --config kubeadmcfg.yaml
52-
kubectl --kubeconfig=/etc/kubernetes/controller-manager.conf config set clusters.kubernetes.server "https://${FULL_NAME}-apiserver:6443"
52+
kubectl --kubeconfig=/etc/kubernetes/controller-manager.conf config set-cluster kubernetes --server "https://${FULL_NAME}-apiserver:6443"
5353
kubectl create secret generic "${FULL_NAME}-controller-manager-conf" --from-file=/etc/kubernetes/controller-manager.conf --dry-run=client -o yaml | kubectl apply -f -
5454
{{- end }}
5555

5656
{{- if .Values.scheduler.enabled }}{{"\n"}}
5757
# generate scheduler kubeconfig
5858
rm -f /etc/kubernetes/scheduler.conf
5959
kubeadm init phase kubeconfig scheduler --config kubeadmcfg.yaml
60-
kubectl --kubeconfig=/etc/kubernetes/scheduler.conf config set clusters.kubernetes.server "https://${FULL_NAME}-apiserver:6443"
60+
kubectl --kubeconfig=/etc/kubernetes/scheduler.conf config set-cluster kubernetes --server "https://${FULL_NAME}-apiserver:6443"
6161
kubectl create secret generic "${FULL_NAME}-scheduler-conf" --from-file=/etc/kubernetes/scheduler.conf --dry-run=client -o yaml | kubectl apply -f -
6262
{{- end }}
6363

64+
{{- if .Values.konnectivityServer.enabled }}{{"\n"}}
65+
# generate konnectivity-server kubeconfig
66+
openssl req -subj "/CN=system:konnectivity-server" -new -newkey rsa:2048 -nodes -out konnectivity.csr -keyout konnectivity.key -out konnectivity.csr
67+
openssl x509 -req -in konnectivity.csr -CA /etc/kubernetes/pki/ca.crt -CAkey /etc/kubernetes/pki/ca.key -CAcreateserial -out konnectivity.crt -days 375 -sha256
68+
kubectl --kubeconfig /etc/kubernetes/konnectivity-server.conf config set-credentials system:konnectivity-server --client-certificate konnectivity.crt --client-key konnectivity.key --embed-certs=true
69+
kubectl --kubeconfig /etc/kubernetes/konnectivity-server.conf config set-cluster kubernetes --server "https://${FULL_NAME}-apiserver:6443" --certificate-authority /etc/kubernetes/pki/ca.crt --embed-certs=true
70+
kubectl --kubeconfig /etc/kubernetes/konnectivity-server.conf config set-context system:konnectivity-server@kubernetes --cluster kubernetes --user system:konnectivity-server
71+
kubectl --kubeconfig /etc/kubernetes/konnectivity-server.conf config use-context system:konnectivity-server@kubernetes
72+
kubectl create secret generic "${FULL_NAME}-konnectivity-server-conf" --from-file=/etc/kubernetes/konnectivity-server.conf --dry-run=client -o yaml | kubectl apply -f -
73+
{{- end }}
74+
6475
# wait for cluster
6576
echo "Waiting for api-server endpoint ${FULL_NAME}-apiserver:6443..."
6677
until kubectl --kubeconfig /etc/kubernetes/admin.conf cluster-info >/dev/null 2>/dev/null; do
@@ -91,6 +102,21 @@ kubectl --kubeconfig "$tmp/kubeconfig" config set clusters..certificate-authorit
91102
kubectl create configmap cluster-info --from-file="$tmp/kubeconfig" --dry-run=client -o yaml | kubectl --kubeconfig /etc/kubernetes/admin.conf apply -n kube-public -f -
92103
rm -rf "$tmp"
93104

105+
{{- if .Values.konnectivityServer.enabled }}{{"\n"}}
106+
# install konnectivity server
107+
kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /manifests/konnectivity-server-rbac.yaml
108+
{{- else }}{{"\n"}}
109+
kubectl --kubeconfig /etc/kubernetes/admin.conf delete clusterrolebinding/system:konnectivity-server 2>/dev/null || true
110+
{{- end }}
111+
112+
{{- if .Values.konnectivityAgent.enabled }}{{"\n"}}
113+
# install konnectivity agent
114+
kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /manifests/konnectivity-agent-deployment.yaml -f /manifests/konnectivity-agent-rbac.yaml
115+
{{- else }}{{"\n"}}
116+
# uninstall konnectivity agent
117+
kubectl --kubeconfig /etc/kubernetes/admin.conf -n kube-system delete deployment/konnectivity-agent serviceaccount/konnectivity-agent 2>/dev/null || true
118+
{{- end }}
119+
94120
{{- if .Values.coredns.enabled }}{{"\n"}}
95121
# install coredns addon
96122
kubeadm init phase addon coredns --config /config/kubeadmcfg.yaml

‎deploy/helm/kubernetes/templates/apiserver-config.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
{{- $fullName := include "kubernetes.fullname" . -}}
2+
---
3+
apiVersion: v1
4+
kind: ConfigMap
5+
metadata:
6+
name: {{ $fullName }}-apiserver-config
7+
data:
8+
egress-selector-configuration.yaml: |
9+
apiVersion: apiserver.k8s.io/v1beta1
10+
kind: EgressSelectorConfiguration
11+
egressSelections:
12+
- name: cluster
13+
connection:
14+
{{- if and .Values.konnectivityServer.enabled }}
15+
proxyProtocol: HTTPConnect
16+
transport:
17+
tcp:
18+
url: "https://{{ $fullName }}-konnectivity-server:8131"
19+
TLSConfig:
20+
caBundle: /pki/konnectivity-client/ca.crt
21+
clientKey: /pki/konnectivity-client/tls.key
22+
clientCert: /pki/konnectivity-client/tls.crt
23+
{{- else }}
24+
proxyProtocol: Direct
25+
{{- end }}
26+
- name: master
27+
connection:
28+
proxyProtocol: Direct
29+
- name: etcd
30+
connection:
31+
proxyProtocol: Direct

‎deploy/helm/kubernetes/templates/apiserver-deployment.yaml‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,11 @@ spec:
2626
{{- with .Values.apiServer.podLabels }}
2727
{{- toYaml . | nindent 8 }}
2828
{{- end }}
29-
{{- with .Values.apiServer.podAnnotations }}
3029
annotations:
30+
checksum/config: {{ include (print $.Template.BasePath "/apiserver-config.yaml") . | sha256sum }}
31+
{{- with .Values.apiServer.podAnnotations }}
3132
{{- toYaml . | nindent 8 }}
32-
{{- end }}
33+
{{- end }}
3334
spec:
3435
{{- with .Values.apiServer.nodeSelector }}
3536
nodeSelector:
@@ -77,6 +78,12 @@ spec:
7778
- --service-cluster-ip-range={{ .Values.apiServer.serviceClusterIPRange }}
7879
- --tls-cert-file=/pki/apiserver/tls.crt
7980
- --tls-private-key-file=/pki/apiserver/tls.key
81+
- --egress-selector-config-file=/etc/kubernetes/egress-selector-configuration.yaml
82+
{{- if .Values.konnectivityAgent.enabled }}{{"\n"}}
83+
- --service-account-issuer=api
84+
- --service-account-signing-key-file=/pki/sa/sa.key
85+
- --api-audiences=system:konnectivity-server
86+
{{- end }}
8087
{{- if not (hasKey .Values.apiServer.extraArgs "advertise-address") }}
8188
{{- with .Values.apiServer.service.loadBalancerIP }}
8289
- --advertise-address={{ . }}
@@ -106,6 +113,8 @@ spec:
106113
{{- toYaml . | nindent 8 }}
107114
{{- end }}
108115
volumeMounts:
116+
- mountPath: /etc/kubernetes
117+
name: apiserver-config
109118
- mountPath: /pki/front-proxy-client
110119
name: pki-front-proxy-client
111120
- mountPath: /pki/apiserver
@@ -116,13 +125,20 @@ spec:
116125
name: pki-apiserver-kubelet-client
117126
- mountPath: /pki/sa
118127
name: pki-sa
128+
{{- if .Values.konnectivityServer.enabled }}{{"\n"}}
129+
- mountPath: /pki/konnectivity-client
130+
name: pki-konnectivity-client
131+
{{- end }}
119132
{{- with .Values.apiServer.extraVolumeMounts }}
120133
{{- toYaml . | nindent 8 }}
121134
{{- end }}
122135
{{- with .Values.apiServer.sidecars }}
123136
{{- toYaml . | nindent 6 }}
124137
{{- end }}
125138
volumes:
139+
- configMap:
140+
name: "{{ $fullName }}-apiserver-config"
141+
name: apiserver-config
126142
- secret:
127143
secretName: "{{ $fullName }}-pki-front-proxy-client"
128144
name: pki-front-proxy-client
@@ -141,6 +157,11 @@ spec:
141157
- secret:
142158
secretName: "{{ $fullName }}-pki-sa"
143159
name: pki-sa
160+
{{- if .Values.konnectivityServer.enabled }}{{"\n"}}
161+
- secret:
162+
secretName: "{{ $fullName }}-pki-konnectivity-client"
163+
name: pki-konnectivity-client
164+
{{- end }}
144165
{{- with .Values.apiServer.extraVolumes }}
145166
{{- toYaml . | nindent 6 }}
146167
{{- end }}

‎deploy/helm/kubernetes/templates/apiserver-service.yaml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,13 @@ metadata:
1010
{{- with .Values.apiServer.service.labels }}
1111
{{- toYaml . | nindent 4 }}
1212
{{- end }}
13-
{{- with .Values.apiServer.service.annotations }}
1413
annotations:
14+
{{- if not (index .Values.apiServer.service.annotations "metallb.universe.tf/allow-shared-ip") }}
15+
metallb.universe.tf/allow-shared-ip: {{ $fullName }}
16+
{{- end }}
17+
{{- with .Values.apiServer.service.annotations }}
1518
{{- toYaml . | nindent 4 }}
16-
{{- end }}
19+
{{- end }}
1720
spec:
1821
type: {{ .Values.apiServer.service.type }}
1922
{{- with .Values.apiServer.service.loadBalancerIP }}

‎deploy/helm/kubernetes/templates/konnectivity-certs.yaml‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
{{- if and .Values.konnectivityServer.enabled }}
2+
{{- $fullName := include "kubernetes.fullname" . -}}
3+
---
4+
apiVersion: cert-manager.io/v1
5+
kind: Certificate
6+
metadata:
7+
name: "{{ $fullName }}-pki-konnectivity-ca"
8+
spec:
9+
commonName: "{{ $fullName }}-pki-konnectivity-ca"
10+
secretName: "{{ $fullName }}-pki-konnectivity-ca"
11+
duration: 87600h # 3650d
12+
renewBefore: 8760h # 365d
13+
subject:
14+
organizations:
15+
- "{{ $fullName }}"
16+
usages:
17+
- "signing"
18+
- "key encipherment"
19+
- "cert sign"
20+
isCA: true
21+
issuerRef:
22+
name: "{{ $fullName }}-selfsigning-issuer"
23+
kind: Issuer
24+
---
25+
apiVersion: cert-manager.io/v1
26+
kind: Issuer
27+
metadata:
28+
name: "{{ $fullName }}-konnectivity-issuer"
29+
spec:
30+
ca:
31+
secretName: "{{ $fullName }}-pki-konnectivity-ca"
32+
---
33+
{{- $svcName1 := printf "%s-konnectivity-server" $fullName }}
34+
{{- $svcName2 := printf "%s-konnectivity-server.%s" $fullName .Release.Namespace }}
35+
{{- $svcName3 := printf "%s-konnectivity-server.%s.svc" $fullName .Release.Namespace }}
36+
{{- $podName1 := printf "*.%s-konnectivity-server" $fullName }}
37+
{{- $podName2 := printf "*.%s-konnectivity-server.%s" $fullName .Release.Namespace }}
38+
{{- $podName3 := printf "*.%s-konnectivity-server.%s.svc" $fullName .Release.Namespace }}
39+
apiVersion: cert-manager.io/v1
40+
kind: Certificate
41+
metadata:
42+
name: "{{ $fullName }}-pki-konnectivity-server"
43+
spec:
44+
commonName: "{{ $fullName }}-pki-konnectivity-server"
45+
secretName: "{{ $fullName }}-pki-konnectivity-server"
46+
duration: 8760h # 365d
47+
renewBefore: 4380h # 178d
48+
subject:
49+
organizations:
50+
- "{{ $fullName }}"
51+
usages:
52+
- "signing"
53+
- "key encipherment"
54+
- "server auth"
55+
dnsNames:
56+
- "{{ $svcName1 }}"
57+
- "{{ $svcName2 }}"
58+
- "{{ $svcName3 }}"
59+
- "{{ $podName1 }}"
60+
- "{{ $podName2 }}"
61+
- "{{ $podName3 }}"
62+
- "localhost"
63+
ipAddresses:
64+
- "127.0.0.1"
65+
issuerRef:
66+
name: "{{ $fullName }}-konnectivity-issuer"
67+
kind: Issuer
68+
---
69+
apiVersion: cert-manager.io/v1
70+
kind: Certificate
71+
metadata:
72+
name: "{{ $fullName }}-pki-konnectivity-client"
73+
spec:
74+
commonName: "{{ $fullName }}-pki-konnectivity-client"
75+
secretName: "{{ $fullName }}-pki-konnectivity-client"
76+
duration: 8760h # 365d
77+
renewBefore: 4380h # 178d
78+
subject:
79+
organizations:
80+
- "system:masters"
81+
usages:
82+
- "signing"
83+
- "key encipherment"
84+
- "client auth"
85+
issuerRef:
86+
name: "{{ $fullName }}-konnectivity-issuer"
87+
kind: Issuer
88+
{{- end }}

0 commit comments

Comments
 (0)