Switch to shared socket for calling buildkitd

welteki · alexellis · commit 7f78acaa9b97 · 2025-10-17T10:20:23.000+01:00
Use a shared socket for communication between Buildkit and the Pro
Builder containers. This simplifies the deployment and removes the need
to create and manage mTLS certificates.

Chart changes:
- `securityContext` for buildkit does not need to be set explicitly in
the values.yaml file anymore. The appropriate default values are
selected based on the value of the `buildkit.rootless` parameter.

- BREAKING CHANGE - The `buildkit.image` parameter is now only used to
set the image when `buildkit.rootless` is `false`. A new parameter
`buildkitRootless.image` is used to set the image for rootless mode.

Signed-off-by: Han Verstraete (OpenFaaS Ltd) &lt;han@openfaas.com&gt;
diff --git a/chart/pro-builder/README.md b/chart/pro-builder/README.md
@@ -13,7 +13,7 @@ The *Function Builder* aka [pro-builder](https://docs.openfaas.com/openfaas-pro/
 - A container image registry that is accessible from your cluster
 
   You can generate a valid container registry login file by:
-  
+
   * Running `faas-cli registry-login` (preferred)
   * Or, disable the keychain in Docker, then run `docker login`, and supply the `$HOME/.docker/config.json` file.
 
@@ -70,33 +70,6 @@ kubectl create secret generic payload-secret \
   --from-file payload-secret=payload.txt -n openfaas
 ```
 
-### mTLS certificates
-
-Generate mTLS certificates for BuildKit and the Pro Builder which are used to encrypt messages between the builder component and BuildKit.
-
-```bash
-docker run -v `pwd`/out:/tmp/ -ti ghcr.io/openfaas/certgen:latest
-
-# Reset the permissions of the files to your own user:
-sudo chown -R $USER:$USER out
-```
-
-Then create two secrets, one for the BuildKit daemon and one for the builder component.
-
-```bash
-kubectl create secret generic -n openfaas \
-  buildkit-daemon-certs \
-  --from-file ./out/certs/ca.crt \
-  --from-file ./out/certs/server.crt \
-  --from-file ./out/certs/server.key
-
-kubectl create secret generic -n openfaas \
-  buildkit-client-certs \
-  --from-file ./out/certs/ca.crt \
-  --from-file ./out/certs/client.crt \
-  --from-file ./out/certs/client.key
-```
-
 ## Install the Chart
 
 - Create the required secret with your OpenFaaS Pro license code:
@@ -123,28 +96,14 @@ Root mode, for development, or where rootless for some reason isn't working:
 
 ```yaml
 buildkit:
-  image: moby/buildkit:v0.10.3
   rootless: false
-  securityContext:
-    runAsUser: 0
-    runAsGroup: 0
-    privileged: true
 ```
 
 Rootless mode (preferred, if possible):
 
 ```yaml
 buildkit:
-  # For a rootless configuration
-  image: moby/buildkit:master-rootless
   rootless: true
-  securityContext:
-    # Needs Kubernetes >= 1.19
-    seccompProfile:
-      type: Unconfined
-    runAsUser: 1000
-    runAsGroup: 1000
-    privileged: false
 ```
 
 Then install the chart using its official path and the custom YAML file:
@@ -346,8 +305,9 @@ Additional pro-builder options in `values.yaml`.
 | `replicas`                | How many replicas of buildkit and the pro-builder API to create                        | `1`                            |
 | `proBuilder.image`        | Container image to use for the pro-builder                                             | See values.yaml                |
 | `proBuilder.maxInflight`  | Limit the total amount of concurrent builds for the  pro-builder replica               | See values.yaml                |
-| `buildkit.image`          | Image version for the buildkit daemon                                                  | See values.yaml                |
-| `buildkit.rootless`       | When set to true, uses user-namespaces to avoid a privileged container                 | See notes in values.yaml       |
+| `buildkit.image`          | Image version for the buildkit daemon when `buildkit.rootless` is false                | See values.yaml                |
+| `buildkitRootless.image`  | Image version for the buildkit daemon when `buildkit.rootless` is true                 | See values.yaml                |
+| `buildkit.rootless`       | When set to true, uses user-namespaces to avoid a privileged container                 | `true`                         |
 | `buildkit.securityContext` | Used to set security policy for buildkit                                              | See values.yaml                |
 | `imagePullPolicy`         | The policy for pulling either of the containers deployed by this chart                 | `IfNotPresent`                 |
 | `disableHmac`             | This setting disable request verification, so should never to set to `true`            | `false`                        |
diff --git a/chart/pro-builder/templates/deployment.yml b/chart/pro-builder/templates/deployment.yml
@@ -40,12 +40,6 @@ spec:
         {{- toYaml .Values.securityContext | nindent 8 }}
 {{- end }}
       volumes:
-        - name: client-certs
-          secret:
-            secretName: buildkit-client-certs
-        - name: daemon-certs
-          secret:
-            secretName: buildkit-daemon-certs
         - name: registry-secret
           secret:
             defaultMode: 420
@@ -67,18 +61,17 @@ spec:
           secret:
             secretName: {{ .Values.awsCredentialsSecret }}
         {{- end}}
-{{- if .Values.serviceAccount}}
+        - name: socket-dir
+          emptyDir: {}
+      {{- if .Values.serviceAccount}}
       serviceAccountName: {{ .Values.serviceAccount | quote }}
-{{- end }}
+      {{- end }}
       containers:
       - name: pro-builder
         image: {{ .Values.proBuilder.image }}
         imagePullPolicy: {{ .Values.imagePullPolicy }}
         args:
           - "-license-file=/var/secrets/license/license"
-          - "--tlscert=/var/secrets/certs/client.crt"
-          - "--tlskey=/var/secrets/certs/client.key"
-          - "--tlscacert=/var/secrets/certs/ca.crt"
         env:
           - name: buildkit_server_name
             value: "127.0.0.1"
@@ -87,7 +80,7 @@ spec:
           - name: insecure
             value: "false"
           - name: buildkit_url
-            value: "tcp://127.0.0.1:1234"
+            value: "unix:///home/app/.local/run/buildkit/buildkitd.sock"
           - name: "disable_hmac"
             value: {{ .Values.disableHmac | quote }}
           - name: "max_inflight"
@@ -118,8 +111,13 @@ spec:
         {{- with .Values.proBuilder.securityContext }}
         securityContext:
           {{- . | toYaml | nindent 12 }}
+        {{- else }}
+        securityContext:
+          runAsGroup: 1000
         {{- end }}
         volumeMounts:
+        - name: socket-dir
+          mountPath: /home/app/.local/run/buildkit
         - name: registry-secret
           readOnly: true
           mountPath: "/home/app/.docker/"
@@ -132,24 +130,25 @@ spec:
         - name: builder-workspace
           mountPath: /tmp/
           readOnly: false
-        - name: client-certs
-          readOnly: true
-          mountPath: /var/secrets/certs
         {{- if .Values.awsCredentialsSecret }}
         - name: aws-credentials
           readOnly: true
           mountPath: "/var/secrets/aws-credentials"
         {{- end }}
       - name: buildkit
         args:
-          - "--addr=tcp://127.0.0.1:1234"
-          - "--tlscert=/var/secrets/certs/server.crt"
-          - "--tlskey=/var/secrets/certs/server.key"
-          - "--tlscacert=/var/secrets/certs/ca.crt"
-{{- if .Values.buildkit.rootless }}
+        {{- if .Values.buildkit.rootless }}
+          - "--addr=unix:///home/user/.local/run/buildkit/buildkitd.sock"
           - "--oci-worker-no-process-sandbox"
-{{- end }}
+        {{- else }}
+          - "--addr=unix:///run/buildkit/buildkitd.sock"
+          - "--group=1000"
+        {{- end }}
+        {{- if .Values.buildkit.rootless }}
+        image: {{ .Values.buildkitRootless.image }}
+        {{- else }}
         image: {{ .Values.buildkit.image }}
+        {{- end }}
         imagePullPolicy: {{ .Values.imagePullPolicy | quote }}
         ports:
         - containerPort: 1234
@@ -159,11 +158,28 @@ spec:
         {{- with .Values.buildkit.securityContext }}
         securityContext:
           {{- . | toYaml | nindent 12 }}
+        {{- else }}
+       {{- if .Values.buildkit.rootless }}
+        securityContext:
+          seccompProfile:
+            type: Unconfined
+          runAsUser: 1000
+          runAsGroup: 1000
+          privileged: false
+        {{- else }}
+        securityContext:
+          runAsUser: 0
+          runAsGroup: 0
+          privileged: true
+        {{- end }}
         {{- end }}
         volumeMounts:
-        - name: daemon-certs
-          readOnly: true
-          mountPath: /var/secrets/certs
+        - name: socket-dir
+          {{- if .Values.buildkit.rootless }}
+          mountPath: /home/user/.local/run/buildkit
+          {{- else }}
+          mountPath: /run/buildkit/
+          {{- end }}
         - name: buildkit-workspace
           mountPath: /tmp/
           readOnly: false
diff --git a/chart/pro-builder/values.yaml b/chart/pro-builder/values.yaml
@@ -19,7 +19,7 @@ proBuilder:
   image: ghcr.io/openfaasltd/pro-builder:0.5.3
 
   # Set to 0 for unlimited, or some non-zero value for a hard limit
-  # the builder will return a HTTP 429 status code, then the client 
+  # the builder will return a HTTP 429 status code, then the client
   # must retry the request.
   # A function executed via the async queue will be retried, so can be
   # a convenient way to build functions without blocking the client.
@@ -32,20 +32,24 @@ proBuilder:
     # limits:
     #   memory: "256Mi"
 
+buildkitRootless:
+  # Image used when running buildkit in rootless mode.
+  image: moby/buildkit:v0.23.2-rootless
+
 # buildkit.image is for the buildkit daemon
 # Check for the latest release on GitHub: https://github.com/moby/buildkit/releases
 #
 # Both configurations are "rootless", however the rootless: true mode does not
 # require Buildkit to run as a privileged container and is preferred.
 buildkit:
-  # A configuration which uses a privileged container for when 
+  # A configuration which uses a privileged container for when
   # your nodes have issues running in rootless mode
   #
-  # Use rootless if possible, and if not, set up a dedicated 
+  # Use rootless if possible, and if not, set up a dedicated
   # nodepool for the function builder pods, which is recycled often
   # through the use of spot instances or preemptive VMs.
   #
-  # image: moby/buildkit:v0.23.2-rootless
+  # image: moby/buildkit:v0.23.2
   # rootless: false
   # securityContext:
   #   runAsUser: 0
@@ -54,16 +58,9 @@ buildkit:
 
   # For a rootless configuration, preferred, if the configuration
   # and Kernel version of your Kubernetes nodes supports it
-  # 
-  image: moby/buildkit:v0.23.2-rootless
+  #
+  image: moby/buildkit:v0.23.2
   rootless: true
-  securityContext:
-    # Needs Kubernetes >= 1.19
-    seccompProfile:
-      type: Unconfined
-    runAsUser: 1000
-    runAsGroup: 1000
-    privileged: false
 
   resources:
     requests:
@@ -97,4 +94,3 @@ nodeSelector: {}
 tolerations: []
 
 affinity: {}
-
