add autoscaler for ingesters (grafana#182)

Signed-off-by: Tom Hayward <thayward@infoblox.com>

Author: kd7lxl

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## master / unreleased
 
+* [FEATURE] Add autoscaler for ingesters #182
 * [ENHANCEMENT] Use FQDN for memcached addresses #175
 
 ## 0.6.0 / 2021-06-28

README.md

@@ -458,6 +458,12 @@ Kubernetes: `^1.19.0-0`
 | ingester.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey | string | `"kubernetes.io/hostname"` |  |
 | ingester.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].weight | int | `100` |  |
 | ingester.annotations | object | `{}` |  |
+| ingester.autoscaling.behavior.scaleDown.selectPolicy | string | `"Disabled"` | Scaledown procedure varies, so automatic scaledown is disabled Ref: https://cortexmetrics.io/docs/guides/ingesters-scaling-up-and-down/#scaling-down |
+| ingester.autoscaling.behavior.scaleUp.policies | list | `[{"periodSeconds":1800,"type":"Pods","value":1}]` | This default scaleup policy allows adding 1 pod every 30 minutes. Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior |
+| ingester.autoscaling.enabled | bool | `false` |  |
+| ingester.autoscaling.maxReplicas | int | `30` |  |
+| ingester.autoscaling.minReplicas | int | `3` |  |
+| ingester.autoscaling.targetMemoryUtilizationPercentage | int | `80` |  |
 | ingester.containerSecurityContext.enabled | bool | `true` |  |
 | ingester.containerSecurityContext.readOnlyRootFilesystem | bool | `true` |  |
 | ingester.env | list | `[]` |  |

templates/ingester/ingester-dep.yaml

@@ -9,7 +9,9 @@ metadata:
   annotations:
     {{- toYaml .Values.ingester.annotations | nindent 4 }}
 spec:
+  {{- if not .Values.ingester.autoscaling.enabled }}
   replicas: {{ .Values.ingester.replicas }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "cortex.ingesterSelectorLabels" . | nindent 6 }}

templates/ingester/ingester-hpa.yaml

@@ -0,0 +1,29 @@
+{{- with .Values.ingester.autoscaling -}}
+{{- if .enabled }}
+apiVersion: autoscaling/v2beta2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "cortex.ingesterFullname" $ }}
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    {{- include "cortex.ingesterLabels" $ | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: {{ if $.Values.ingester.statefulSet.enabled }}StatefulSet{{ else }}Deployment{{ end }}
+    name: {{ include "cortex.ingesterFullname" $ }}
+  minReplicas: {{ .minReplicas }}
+  maxReplicas: {{ .maxReplicas }}
+  metrics:
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .targetMemoryUtilizationPercentage }}
+  {{- with .behavior }}
+  behavior:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}
+{{- end }}

templates/ingester/ingester-statefulset.yaml

@@ -9,7 +9,9 @@ metadata:
   annotations:
     {{- toYaml .Values.ingester.annotations | nindent 4 }}
 spec:
+  {{- if not .Values.ingester.autoscaling.enabled }}
   replicas: {{ .Values.ingester.replicas }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "cortex.ingesterSelectorLabels" . | nindent 6 }}

values.yaml

@@ -511,6 +511,24 @@ ingester:
 
   annotations: {}
 
+  autoscaling:
+    enabled: false
+    minReplicas: 3
+    maxReplicas: 30
+    targetMemoryUtilizationPercentage: 80
+    behavior:
+      scaleDown:
+        # -- Scaledown procedure varies, so automatic scaledown is disabled
+        # Ref: https://cortexmetrics.io/docs/guides/ingesters-scaling-up-and-down/#scaling-down
+        selectPolicy: Disabled
+      scaleUp:
+        # -- This default scaleup policy allows adding 1 pod every 30 minutes.
+        # Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior
+        policies:
+          - type: Pods
+            value: 1
+            periodSeconds: 1800
+
   ## DEPRECATED: use persistentVolume.subPath instead
   persistence:
     subPath: