Add taintmanager

Author: xcompass

jupyterhub/templates/image-puller/_helpers-daemonset.tpl

@@ -51,14 +51,16 @@ spec:
         per node limit all k8s clusters have and have a higher priority
         than user-placeholder pods that could block an entire node.
       */}}
+      serviceAccount: taintmanager
+      serviceAccountName: taintmanager
       {{- if .Values.scheduling.podPriority.enabled }}
       priorityClassName: {{ include "jupyterhub.image-puller-priority.fullname" . }}
       {{- end }}
       {{- with .Values.singleuser.nodeSelector }}
       nodeSelector:
         {{- . | toYaml | nindent 8 }}
       {{- end }}
-      {{- with concat .Values.scheduling.userPods.tolerations .Values.singleuser.extraTolerations .Values.prePuller.extraTolerations }}
+      {{- with concat .Values.scheduling.userPods.tolerations .Values.singleuser.extraTolerations .Values.prePuller.extraTolerations .Values.prePuller.taintmanager.tolerations }}
       tolerations:
         {{- . | toYaml | nindent 8 }}
       {{- end }}
@@ -70,11 +72,39 @@ spec:
               {{- include "jupyterhub.userNodeAffinityRequired" . | nindent 14 }}
       {{- end }}
       terminationGracePeriodSeconds: 0
-      automountServiceAccountToken: false
+      automountServiceAccountToken: true
       {{- with include "jupyterhub.imagePullSecrets" (dict "root" . "image" .Values.singleuser.image) }}
       imagePullSecrets: {{ . }}
       {{- end }}
       initContainers:
+        {{- if .Values.prePuller.taintmanager.enabled }}
+        {{- $taint := first .Values.prePuller.taintmanager.tolerations }}
+        - name: taintmanager-adding
+          image: {{ .Values.prePuller.taintmanager.image.name }}:{{ .Values.prePuller.taintmanager.image.tag }}
+          command:
+            - /taintmanager
+            - -add
+            - {{ $taint.key }}:{{ $taint.effect }}
+          env:
+            - name: GODEBUG
+              value: x509sha1=1
+            - name: MY_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: MY_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          {{- with .Values.prePuller.resources }}
+          resources:
+            {{- . | toYaml | nindent 12 }}
+          {{- end }}
+          {{- with .Values.prePuller.containerSecurityContext }}
+          securityContext:
+            {{- . | toYaml | nindent 12 }}
+          {{- end }}
+        {{- end }}
         {{- /* --- Conditionally pull an image all user pods will use in an initContainer --- */}}
         {{- $blockWithIptables := hasKey .Values.singleuser.cloudMetadata "enabled" | ternary (not .Values.singleuser.cloudMetadata.enabled) .Values.singleuser.cloudMetadata.blockWithIptables }}
         {{- if $blockWithIptables }}
@@ -172,6 +202,34 @@ spec:
             {{- . | toYaml | nindent 12 }}
           {{- end }}
         {{- end }}
+        {{- if .Values.prePuller.taintmanager.enabled }}
+        {{- $taint := first .Values.prePuller.taintmanager.tolerations }}
+        - name: taintmanager-removing
+          image: {{ .Values.prePuller.taintmanager.image.name }}:{{ .Values.prePuller.taintmanager.image.tag }}
+          command:
+            - /taintmanager
+            - -remove
+            - {{ $taint.key }}:{{ $taint.effect }}
+          env:
+            - name: GODEBUG
+              value: x509sha1=1
+            - name: MY_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: MY_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          {{- with .Values.prePuller.resources }}
+          resources:
+            {{- . | toYaml | nindent 12 }}
+          {{- end }}
+          {{- with .Values.prePuller.containerSecurityContext }}
+          securityContext:
+            {{- . | toYaml | nindent 12 }}
+          {{- end }}
+        {{- end }}
       containers:
         - name: pause
           image: {{ .Values.prePuller.pause.image.name }}:{{ .Values.prePuller.pause.image.tag }}

jupyterhub/templates/image-puller/rbac.yaml

@@ -42,4 +42,40 @@ roleRef:
   name: {{ include "jupyterhub.hook-image-awaiter.fullname" . }}
   apiGroup: rbac.authorization.k8s.io
 {{- end }}
+{{- if .Values.prePuller.taintmanager.enabled }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: taintmanager
+  namespace: "{{ .Release.Namespace }}"
+  labels:
+    {{- include "jupyterhub.labels" . | nindent 4 }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: taintmanager
+  labels:
+    {{- include "jupyterhub.labels" . | nindent 4 }}
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resources: ["nodes"]
+  verbs: ["get", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: taintmanager
+  labels:
+    {{- include "jupyterhub.labels" . | nindent 4 }}
+subjects:
+- kind: ServiceAccount
+  name: taintmanager
+  namespace: "{{ .Release.Namespace }}"
+roleRef:
+  kind: ClusterRole
+  name: taintmanager
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
 {{- end }}

jupyterhub/templates/scheduling/user-placeholder/statefulset.yaml

@@ -47,7 +47,7 @@ spec:
       nodeSelector:
         {{- . | toYaml | nindent 8 }}
       {{- end }}
-      {{- with concat .Values.scheduling.userPods.tolerations .Values.singleuser.extraTolerations }}
+      {{- with concat .Values.scheduling.userPods.tolerations .Values.singleuser.extraTolerations .Values.prePuller.taintmanager.tolerations }}
       tolerations:
         {{- . | toYaml | nindent 8 }}
       {{- end }}

jupyterhub/values.yaml

@@ -609,6 +609,17 @@ prePuller:
       annotations: {}
   continuous:
     enabled: true
+  taintmanager:
+    enabled: true
+    image:
+      name: lthub/taintmanager
+      tag: master
+    # taint/tolerations used for preventing user pods being scheduled while user image is being pulled
+    # only first element in the list is being used
+    tolerations:
+      - key: hub.jupyter.org/imagepulling
+        operator: Exists
+        effect: NoExecute
   pullProfileListImages: true
   extraImages: {}
   pause: