Incremental cluster growth (#769)

Author: vishalbollu

manager/manifests/cluster-autoscaler.yaml.j2

@@ -170,7 +170,10 @@ spec:
             - --expander=least-waste
             {% endif %}
             - --max-nodes-total={{ config['max_instances'] + 1 }}
+            - --max-total-unready-percentage=5
+            - --ok-total-unready-count=30
             - --max-node-provision-time=5m
+            - --scan-interval=20s
             - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/{{ config['cluster_name'] }}
           volumeMounts:
             - name: ssl-certs

manager/manifests/fluentd.yaml

@@ -47,7 +47,15 @@ roleRef:
   name: fluentd
   apiGroup: rbac.authorization.k8s.io
 ---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: fluentd
+value: 1000
+globalDefault: false
+description: "This priority class should be used for fluentd daemonset only."
 
+---
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -166,6 +174,7 @@ spec:
         app: fluentd
     spec:
       serviceAccountName: fluentd
+      priorityClassName: fluentd
       initContainers:
         - name: copy-fluentd-config
           image: busybox

manager/manifests/nvidia.yaml

@@ -14,6 +14,14 @@
 
 # Source: https://github.com/NVIDIA/k8s-device-plugin/blob/1.0.0-beta4/nvidia-device-plugin.yml
 
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: nvidia
+value: 1000
+globalDefault: false
+description: "This priority class should be used for nvidia daemonset only."
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -34,6 +42,7 @@ spec:
       labels:
         name: nvidia-device-plugin-ds
     spec:
+      priorityClassName: nvidia
       tolerations:
       # This toleration is deprecated. Kept here for backward compatibility
       # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/

manager/manifests/statsd.yaml

@@ -41,6 +41,14 @@ metadata:
   name: cwagentstatsdconfig
   namespace: default
 ---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: statsd
+value: 1000
+globalDefault: false
+description: "This priority class should be used for statsd daemonset only."
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -55,6 +63,7 @@ spec:
       labels:
         name: cloudwatch-agent-statsd
     spec:
+      priorityClassName: statsd
       containers:
         - name: cloudwatch-agent
           image: $CORTEX_IMAGE_STATSD