Merge pull request #49 from ijc/cgroups-improvements

Improved cgroup configuration

Author: ijc

pkg/cri-containerd/build.yml

@@ -16,13 +16,10 @@ config:
   - /run/containerd/containerd.sock:/run/containerd/containerd.sock
   - /var/lib/kubelet-plugins:/usr/libexec/kubernetes/kubelet-plugins:rshared,rbind
   mounts:
-  - type: cgroup
-    options:
-    - rw
-    - nosuid
-    - noexec
-    - nodev
-    - relatime
+  - type: bind
+    source: /sys/fs/cgroup
+    destination: /sys/fs/cgroup
+    options: ["rw","rbind","rshared","nosuid","noexec","nodev","relatime"]
   capabilities:
   - all
   rootfsPropagation: shared

pkg/kubelet/build.yml

@@ -17,18 +17,19 @@ config:
   - /var/lib/cni/conf:/etc/cni/net.d:rshared,rbind
   - /var/lib/cni/bin:/opt/cni/bin:rshared,rbind
   mounts:
-  - type: cgroup
-    options:
-    - rw
-    - nosuid
-    - noexec
-    - nodev
-    - relatime
+  - type: bind
+    source: /sys/fs/cgroup
+    destination: /sys/fs/cgroup
+    options: ["rw","rbind","rshared","nosuid","noexec","nodev","relatime"]
   capabilities:
   - all
   rootfsPropagation: shared
   pid: host
   runtime:
+    cgroups:
+    - systemreserved
+    - podruntime
+    - kubepods
     mkdir:
     - /var/lib/kubeadm
     - /var/lib/cni/conf
@@ -38,12 +39,8 @@ config:
     - type: bind
       source: /var/lib/cni/bin
       destination: /opt/cni/bin
-      options:
-      - rw
-      - bind
+      options: ["rw","bind"]
     - type: bind
       source: /var/lib/cni/conf
       destination: /etc/cni/net.d
-      options:
-      - rw
-      - bind
+      options: ["rw","bind"]

pkg/kubelet/kubelet.sh

@@ -63,6 +63,14 @@ echo "kubelet.sh: ${await} has arrived" 2>&1
 
 mkdir -p /etc/kubernetes/manifests
 
+# If using --cgroups-per-qos then need to use --cgroup-root=/ and not
+# the --cgroup-root=kubepods from below. This can be done at image
+# build time by adding to the service definition:
+#
+#    command:
+#      - /usr/bin/kubelet.sh
+#      - --cgroup-root=/
+#      - --cgroups-per-qos
 exec kubelet --kubeconfig=/etc/kubernetes/kubelet.conf \
 	      --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
 	      --pod-manifest-path=/etc/kubernetes/manifests \
@@ -77,4 +85,5 @@ exec kubelet --kubeconfig=/etc/kubernetes/kubelet.conf \
 	      --cadvisor-port=0 \
 	      --kube-reserved-cgroup=podruntime \
 	      --system-reserved-cgroup=systemreserved \
+	      --cgroup-root=kubepods \
 	      $KUBELET_ARGS $@

yml/cri-containerd.yml

@@ -1,6 +1,6 @@
 services:
   - name: cri-containerd
-    image: linuxkit/cri-containerd:6d7a253f3e69c506c76f0a0dfa32b7c307b0c9ee
+    image: linuxkit/cri-containerd:b16c1a3f11b986d8f818d2797920db411ff51ac1
     cgroupsPath: podruntime/cri-containerd
 files:
   - path: /etc/kubelet.sh.conf

yml/docker.yml

@@ -5,8 +5,10 @@ services:
      - all
     pid: host
     mounts:
-     - type: cgroup
-       options: ["rw","nosuid","noexec","nodev","relatime"]
+     - type: bind
+       source: /sys/fs/cgroup
+       destination: /sys/fs/cgroup
+       options: ["rw","rbind","rshared","nosuid","noexec","nodev","relatime"]
     binds:
      - /dev:/dev
      - /etc/resolv.conf:/etc/resolv.conf

yml/kube.yml

@@ -1,8 +1,8 @@
 kernel:
-  image: linuxkit/kernel:4.9.75
+  image: linuxkit/kernel:4.9.76
   cmdline: "console=tty0 console=ttyS0"
 init:
-  - linuxkit/init:26def2174c74efa0ea8006ebd63ebb5d02e9513e
+  - linuxkit/init:6992bd1308bdfd8af5a74aba293bb53e99b482bd
   - linuxkit/runc:abc3f292653e64a2fd488e9675ace19a55ec7023
   - linuxkit/containerd:e58a382c33bb509ba3e0e8170dfaa5a100504c5b
   - linuxkit/ca-certificates:de21b84d9b055ad9dcecc57965b654a7a24ef8e0
@@ -20,7 +20,7 @@ onboot:
   - name: metadata
     image: linuxkit/metadata:2af15c9f4b0e73515c219b7cc14e6e65e1d4fd6d
   - name: format
-    image: linuxkit/format:e945016ec780a788a71dcddc81497d54d3b14bc7
+    image: linuxkit/format:5a1140cb65e733f26de727fa684fef1326e9d5ab
   - name: mounts
     image: linuxkit/mount:b346ec277b7074e5c9986128a879c10a1d18742b
     command: ["/usr/bin/mountie", "/var/lib/"]
@@ -40,12 +40,8 @@ services:
     image: linuxkit/sshd:ac5e8364e2e9aa8717a3295c51eb60b8c57373d5
     cgroupsPath: systemreserved/sshd
   - name: kubelet
-    image: linuxkit/kubelet:0e6902fbb878f592b8be03605e18d917e622ce31
+    image: linuxkit/kubelet:bf10112fabce0f713ea1eea2a8798d350b25fe98
     cgroupsPath: podruntime/kubelet
-    runtime:
-      cgroups:
-        - systemreserved
-        - podruntime
 files:
   - path: etc/linuxkit.yml
     metadata: yaml