Enable IPVS on kube proxy (#2357)

Author: RobertLucian

dev/versions.md

@@ -18,6 +18,13 @@
 1. Update `ami.json` (see release checklist for instructions)
 1. See instructions for upgrading the Kubernetes client below
 
+## kube-proxy (IPVS mode)
+
+1. Before spinning up a Cortex cluster with the new eksctl/kubernetes/eks updates, make sure to have the `setup_ipvs` functional call commented out in the manager.
+1. Once the cluster is up, run the `cat /var/lib/kube-proxy-config/config` command on any of the kube-proxy pods of the cluster. Compare the output of that with what the `upgrade_kube_proxy_mode.py` script is applying and make sure it's still applicable, if not, check out the spec of the [KubeProxyConfiguration](https://kubernetes.io/docs/reference/config-api/kube-proxy-config.v1alpha1/) and upgrade `upgrade_kube_proxy_mode.py`.
+1. Compare the spec of the `kube-proxy.patch.yaml` patch with the current spec of the kube-proxy daemoset and make sure it's still applicable. You can either inspect the `kube-proxy` command helper by exec-ing into the pod or by looking at the [kube-proxy](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/) documentation for the respective version of Kubernetes.
+1. Once both config map and the daemonset are updated and the kube-proxy pod(s) has/have started, make sure you notice the `Using ipvs Proxier` log.
+
 ## aws-iam-authenticator
 
 1. Find the latest release [here](https://docs.aws.amazon.com/eks/latest/userguide/install-aws-iam-authenticator.html)

manager/generate_eks.py

@@ -67,6 +67,15 @@ def default_nodegroup(cluster_config):
             "evictionHard": {"memory.available": "200Mi", "nodefs.available": "5%"},
             "registryPullQPS": 10,
         },
+        "preBootstrapCommands": [
+            "sudo yum install -y ipvsadm",
+            "sudo modprobe ip_vs",  # IP virtual server
+            "sudo modprobe ip_vs_rr",  # round robing load balancer
+            "sudo modprobe ip_vs_lc",  # least connected load balancer
+            "sudo modprobe ip_vs_wrr",  # weighted round robin load balancer
+            "sudo modprobe ip_vs_sh",  # source-hashing load balancer
+            "sudo modprobe nf_conntrack_ipv4",
+        ],
     }
 
 

manager/install.sh

@@ -41,6 +41,7 @@ function cluster_up() {
   echo "✓"
 
   echo -n "￮ configuring networking (this will take a few minutes) "
+  setup_ipvs
   setup_istio
   python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/apis.yaml.j2 | kubectl apply -f - >/dev/null
   echo "✓"
@@ -368,6 +369,25 @@ function remove_nodegroups() {
   echo
 }
 
+function setup_ipvs() {
+  # get a random kube-proxy pod
+  kubectl rollout status daemonset kube-proxy -n kube-system --timeout 30m >/dev/null
+  kube_proxy_pod=$(kubectl get pod -n kube-system -l k8s-app=kube-proxy -o jsonpath='{.items[*].metadata.name}' | cut -d " " -f1)
+
+  # export kube-proxy's current config
+  kubectl exec -it -n kube-system ${kube_proxy_pod} -- cat /var/lib/kube-proxy-config/config > proxy_config.yaml
+
+  # upgrade proxy mode from the exported kube-proxy config
+  python upgrade_kube_proxy_mode.py proxy_config.yaml > upgraded_proxy_config.yaml
+
+  # update kube-proxy's configmap to include the updated configuration
+  kubectl get configmap -n kube-system kube-proxy -o yaml | yq --arg replace "`cat upgraded_proxy_config.yaml`" '.data.config=$replace' | kubectl apply -f - >/dev/null
+
+  # patch the kube-proxy daemonset
+  kubectl patch ds -n kube-system kube-proxy --patch "$(cat manifests/kube-proxy.patch.yaml)" >/dev/null
+  kubectl rollout status daemonset kube-proxy -n kube-system --timeout 30m >/dev/null
+}
+
 function setup_istio() {
   if ! grep -q "istio-customgateway-certs" <<< $(kubectl get secret -n istio-system); then
     WEBSITE=localhost

manager/manifests/kube-proxy.patch.yaml

@@ -0,0 +1,42 @@
+# Copyright 2021 Cortex Labs, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a patch that needs to be applied onto the daemonset that's added by eksctl.
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-proxy
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: kube-proxy
+  template:
+    spec:
+      containers:
+      - name: kube-proxy
+        command:
+        - kube-proxy
+        - --v=2
+        - --proxy-mode=ipvs
+        - --ipvs-scheduler=rr
+        - --config=/var/lib/kube-proxy/config
+        env:
+        - name: KUBE_PROXY_MODE
+          value: ipvs
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 20%
+    type: RollingUpdate

manager/upgrade_kube_proxy_mode.py

@@ -0,0 +1,33 @@
+# Copyright 2021 Cortex Labs, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Usage: python create_user.py $KUBE_PROXY_CONFIG.yaml
+
+import yaml
+import sys
+
+
+def main():
+    kube_proxy_config_file = sys.argv[1]
+    with open(kube_proxy_config_file, "r") as f:
+        kube_proxy_config = yaml.safe_load(f)
+
+    kube_proxy_config["mode"] = "ipvs"  # IP Virtual Server
+    kube_proxy_config["ipvs"]["scheduler"] = "rr"  # round robin
+
+    print(yaml.dump(kube_proxy_config, indent=2))
+
+
+if __name__ == "__main__":
+    main()