add autoscaler for distributors (grafana#189)

* add autoscaler for distributors

Signed-off-by: Tom Hayward <thayward@infoblox.com>

* enable autoscaling in the CI

It is not expected that the autoscaling will trigger in the CI, but including it will validate the manifest.

Signed-off-by: Tom Hayward <thayward@infoblox.com>

Author: kd7lxl

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## master / unreleased
 
+* [FEATURE] Add autoscaler for distributors #189
 * [FEATURE] Add autoscaler for ingesters #182
 * [ENHANCEMENT] Define namespace in templates #184
 * [ENHANCEMENT] Use FQDN for memcached addresses #175

README.md

@@ -413,6 +413,12 @@ Kubernetes: `^1.19.0-0`
 | distributor.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey | string | `"kubernetes.io/hostname"` |  |
 | distributor.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].weight | int | `100` |  |
 | distributor.annotations | object | `{}` |  |
+| distributor.autoscaling.behavior | object | `{}` | Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior |
+| distributor.autoscaling.enabled | bool | `false` | Creates a HorizontalPodAutoscaler for the distributor pods. |
+| distributor.autoscaling.maxReplicas | int | `30` |  |
+| distributor.autoscaling.minReplicas | int | `2` |  |
+| distributor.autoscaling.targetCPUUtilizationPercentage | int | `80` |  |
+| distributor.autoscaling.targetMemoryUtilizationPercentage | int | `0` |  |
 | distributor.containerSecurityContext.enabled | bool | `true` |  |
 | distributor.containerSecurityContext.readOnlyRootFilesystem | bool | `true` |  |
 | distributor.env | list | `[]` |  |

ci/test-values.yaml

@@ -57,9 +57,15 @@ config:
 
 ingester:
   replicas: 1
+  autoscaling:
+    enabled: true
+    minReplicas: 1
   statefulSet:
     enabled: true
 distributor:
   replicas: 1
+  autoscaling:
+    enabled: true
+    minReplicas: 1
 nginx:
   replicas: 1

templates/distributor/distributor-dep.yaml

@@ -9,7 +9,9 @@ metadata:
   annotations:
     {{- toYaml .Values.distributor.annotations | nindent 4 }}
 spec:
+  {{- if not .Values.distributor.autoscaling.enabled }}
   replicas: {{ .Values.distributor.replicas }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "cortex.distributorSelectorLabels" . | nindent 6 }}

templates/distributor/distributor-hpa.yaml

@@ -0,0 +1,39 @@
+{{- with .Values.distributor.autoscaling -}}
+{{- if .enabled }}
+apiVersion: autoscaling/v2beta2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "cortex.distributorFullname" $ }}
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    {{- include "cortex.distributorLabels" $ | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "cortex.distributorFullname" $ }}
+  minReplicas: {{ .minReplicas }}
+  maxReplicas: {{ .maxReplicas }}
+  metrics:
+  {{- with .targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ . }}
+  {{- end }}
+  {{- with .targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ . }}
+  {{- end }}
+  {{- with .behavior }}
+  behavior:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}
+{{- end }}

values.yaml

@@ -422,6 +422,17 @@ distributor:
             topologyKey: 'kubernetes.io/hostname'
 
   annotations: {}
+
+  autoscaling:
+    # -- Creates a HorizontalPodAutoscaler for the distributor pods.
+    enabled: false
+    minReplicas: 2
+    maxReplicas: 30
+    targetCPUUtilizationPercentage: 80
+    targetMemoryUtilizationPercentage: 0  # 80
+    # -- Ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior
+    behavior: {}
+
   persistence:
     subPath: