Skip to content

Commit cb68f90

Browse files
friedrichgfriedrichg
authored
Configure default GOMAXPROCS and GOMEMLIMIT (#32)
Also remove mem-ballast, that is not required if using GOMEMLIMIT Signed-off-by: Friedrich Gonzalez <friedrichg@gmail.com>
1 parent 3829a81 commit cb68f90

13 files changed

+73
-31
lines changed

‎CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
# Changelog
22

33
## master / unreleased
4+
* [CHANGE] Remove mem-ballast from distributor and querier.
5+
* [CHANGE] Increase cpu requests for querier to 2.
6+
* [CHANGE] Configure default GOMAXPROCS and GOMEMLIMIT for all cortex modules
47
* [CHANGE] Add default tenant shard sizes
58
* [CHANGE] Use cortex v1.15.3
69
* [CHANGE] Azure storage endpoint suffix is set to `blob.core.windows.net` for backward compatibility

‎cortex/alertmanager.libsonnet‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@
9696
if $._config.alertmanager_enabled then
9797
container.new('alertmanager', $._images.alertmanager) +
9898
container.withPorts($.util.defaultPorts + mode.ports) +
99+
container.withEnvMap($.alertmanager_env_map) +
99100
container.withEnvMixin([container.envType.fromFieldPath('POD_IP', 'status.podIP')]) +
100101
container.withArgsMixin(
101102
$.util.mapToFlags($.alertmanager_args) +
@@ -112,6 +113,11 @@
112113
$.jaeger_mixin
113114
else {},
114115

116+
alertmanager_env_map:: {
117+
GOMAXPROCS: '1',
118+
GOMEMLIMIT: '1GiB',
119+
},
120+
115121
alertmanager_statefulset:
116122
if $._config.alertmanager_enabled then
117123
statefulSet.new('alertmanager', $._config.alertmanager.replicas, [$.alertmanager_container], $.alertmanager_pvc) +

‎cortex/compactor.libsonnet‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,19 @@
4343
container.new('compactor', $._images.compactor) +
4444
container.withPorts($.compactor_ports) +
4545
container.withArgsMixin($.util.mapToFlags($.compactor_args)) +
46+
container.withEnvMap($.compactor_env_map) +
4647
container.withVolumeMountsMixin([volumeMount.new('compactor-data', '/data')]) +
4748
// Do not limit compactor CPU and request enough cores to honor configured max concurrency.
4849
$.util.resourcesRequests($._config.cortex_compactor_max_concurrency, '6Gi') +
4950
$.util.resourcesLimits(null, '6Gi') +
5051
$.util.readinessProbe +
5152
$.jaeger_mixin,
5253

54+
compactor_env_map:: {
55+
GOMAXPROCS: std.toString($._config.cortex_compactor_max_concurrency),
56+
GOMEMLIMIT: '6GiB',
57+
},
58+
5359
newCompactorStatefulSet(name, container)::
5460
statefulSet.new(name, 1, [container], compactor_data_pvc) +
5561
statefulSet.mixin.spec.withServiceName(name) +

‎cortex/distributor.libsonnet‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,6 @@
2020
'distributor.ha-tracker.etcd.endpoints': 'etcd-client.%s.svc.cluster.local.:2379' % $._config.namespace,
2121
'distributor.ha-tracker.prefix': 'prom_ha/',
2222

23-
// The memory requests are 2G, and we barely use 100M.
24-
// By adding a ballast of 1G, we can drastically reduce GC, but also keep the usage at
25-
// around 1.25G, reducing the 99%ile.
26-
'mem-ballast-size-bytes': 1 << 30, // 1GB
27-
2823
'server.grpc.keepalive.max-connection-age': '2m',
2924
'server.grpc.keepalive.max-connection-age-grace': '5m',
3025
'server.grpc.keepalive.max-connection-idle': '1m',
@@ -38,12 +33,18 @@
3833
'distributor.extend-writes': $._config.unregister_ingesters_on_shutdown,
3934
},
4035

36+
distributor_env_map:: {
37+
GOMAXPROCS: '2',
38+
GOMEMLIMIT: '2GiB',
39+
},
40+
4141
distributor_ports:: $.util.defaultPorts,
4242

4343
distributor_container::
4444
container.new('distributor', $._images.distributor) +
4545
container.withPorts($.distributor_ports) +
4646
container.withArgsMixin($.util.mapToFlags($.distributor_args)) +
47+
container.withEnvMap($.distributor_env_map) +
4748
$.util.resourcesRequests('2', '2Gi') +
4849
$.util.resourcesLimits(null, '4Gi') +
4950
$.util.readinessProbe +

‎cortex/flusher-job-blocks.libsonnet‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,17 @@
2121
target: 'flusher',
2222
'blocks-storage.tsdb.retention-period': '10000h', // don't delete old blocks too soon.
2323
})) +
24+
container.withEnvMap($.flusher_env_map) +
2425
$.util.resourcesRequests('4', '15Gi') +
2526
$.util.resourcesLimits(null, '25Gi') +
2627
$.util.readinessProbe +
2728
$.jaeger_mixin,
2829

30+
flusher_env_map:: {
31+
GOMAXPROCS: '4',
32+
GOMEMLIMIT: '15GiB',
33+
},
34+
2935
flusher_job_func(jobName, pvcName)::
3036
job.new() +
3137
job.mixin.spec.template.spec.withContainers([

‎cortex/ingester.libsonnet‎

Lines changed: 7 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
local pvc = $.core.v1.persistentVolumeClaim,
44
local statefulSet = $.apps.v1.statefulSet,
55
local volume = $.core.v1.volume,
6+
local volumeMount = $.core.v1.volumeMount,
67

78
// The ingesters should persist TSDB blocks and WAL on a persistent
89
// volume in order to be crash resilient.
@@ -44,18 +45,6 @@
4445
'ingester.tokens-file-path': '/data/tokens',
4546
},
4647

47-
ingester_statefulset_args::
48-
$._config.grpcConfig
49-
{
50-
'ingester.wal-enabled': true,
51-
'ingester.checkpoint-enabled': true,
52-
'ingester.recover-from-wal': true,
53-
'ingester.wal-dir': $._config.ingester.wal_dir,
54-
'ingester.checkpoint-duration': '15m',
55-
'-log.level': 'info',
56-
'ingester.tokens-file-path': $._config.ingester.wal_dir + '/tokens',
57-
},
58-
5948
ingester_ports:: $.util.defaultPorts,
6049

6150
local name = 'ingester',
@@ -65,22 +54,19 @@
6554
container.new(name, $._images.ingester) +
6655
container.withPorts($.ingester_ports) +
6756
container.withArgsMixin($.util.mapToFlags($.ingester_args)) +
57+
container.withEnvMap($.ingester_env_map) +
6858
$.util.resourcesRequests('4', '15Gi') +
6959
$.util.resourcesLimits(null, '25Gi') +
7060
$.util.readinessProbe +
7161
$.jaeger_mixin,
7262

73-
local volumeMount = $.core.v1.volumeMount,
74-
75-
ingester_statefulset_container::
76-
$.ingester_container +
77-
container.withArgsMixin($.util.mapToFlags($.ingester_statefulset_args)) +
78-
container.withVolumeMountsMixin([
79-
volumeMount.new('ingester-pvc', $._config.ingester.wal_dir),
80-
]),
81-
8263
ingester_deployment_labels:: {},
8364

65+
ingester_env_map:: {
66+
GOMAXPROCS: '4',
67+
GOMEMLIMIT: '15GiB',
68+
},
69+
8470
local ingester_pvc =
8571
pvc.new('ingester-pvc') +
8672
pvc.mixin.spec.resources.withRequests({ storage: $._config.ingester.statefulset_disk }) +

‎cortex/querier.libsonnet‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,14 @@
2626
'querier.frontend-address': 'query-frontend-discovery.%(namespace)s.svc.cluster.local:9095' % $._config,
2727
'querier.frontend-client.grpc-max-send-msg-size': 100 << 20,
2828

29-
// We request high memory but the Go heap is typically very low (< 100MB) and this causes
30-
// the GC to trigger continuously. Setting a ballast of 256MB reduces GC.
31-
'mem-ballast-size-bytes': 1 << 28, // 256M
32-
3329
'log.level': 'debug',
3430
},
3531

3632
querier_ports:: $.util.defaultPorts,
3733

3834
querier_env_map:: {
35+
GOMAXPROCS: '2',
36+
GOMEMLIMIT: '12Gi',
3937
JAEGER_REPORTER_MAX_QUEUE_SIZE: '1024', // Default is 100.
4038
},
4139

@@ -46,7 +44,7 @@
4644
$.jaeger_mixin +
4745
$.util.readinessProbe +
4846
container.withEnvMap($.querier_env_map) +
49-
$.util.resourcesRequests('1', '12Gi') +
47+
$.util.resourcesRequests('2', '12Gi') +
5048
$.util.resourcesLimits(null, '24Gi'),
5149

5250
local deployment = $.apps.v1.deployment,

‎cortex/query-frontend.libsonnet‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,17 @@
4242
container.new('query-frontend', $._images.query_frontend) +
4343
container.withPorts($.util.defaultPorts) +
4444
container.withArgsMixin($.util.mapToFlags($.query_frontend_args)) +
45+
container.withEnvMap($.query_frontend_env_map) +
4546
$.jaeger_mixin +
4647
$.util.readinessProbe +
4748
$.util.resourcesRequests('2', '600Mi') +
4849
$.util.resourcesLimits(null, '1200Mi'),
4950

51+
query_frontend_env_map:: {
52+
GOMAXPROCS: '2',
53+
GOMEMLIMIT: '600MiB',
54+
},
55+
5056
local deployment = $.apps.v1.deployment,
5157

5258
newQueryFrontendDeployment(name, container)::

‎cortex/query-scheduler.libsonnet‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
container.new('query-scheduler', $._images.query_scheduler) +
1818
container.withPorts($.util.defaultPorts) +
1919
container.withArgsMixin($.util.mapToFlags($.query_scheduler_args)) +
20+
container.withEnvMap($.query_scheduler_env_map) +
2021
$.jaeger_mixin +
2122
$.util.readinessProbe +
2223
$.util.resourcesRequests('2', '1Gi') +
@@ -30,6 +31,11 @@
3031
deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(0) +
3132
deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1),
3233

34+
query_scheduler_env_map:: {
35+
GOMAXPROCS: '2',
36+
GOMEMLIMIT: '1GiB',
37+
},
38+
3339
query_scheduler_deployment: if !$._config.query_scheduler_enabled then {} else
3440
self.newQuerySchedulerDeployment('query-scheduler', $.query_scheduler_container),
3541

‎cortex/query-tee.libsonnet‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,15 @@
1818
containerPort.newNamed(name='http-metrics', containerPort=9900),
1919
]) +
2020
container.withArgsMixin($.util.mapToFlags($.query_tee_args)) +
21+
container.withEnvMap($.query_tee_env_map) +
2122
$.util.resourcesRequests('1', '512Mi') +
2223
$.jaeger_mixin,
2324

25+
query_tee_env_map:: {
26+
GOMAXPROCS: '1',
27+
GOMEMLIMIT: '512MiB',
28+
},
29+
2430
query_tee_deployment: if !($._config.query_tee_enabled) then {} else
2531
deployment.new('query-tee', 2, [$.query_tee_container]),
2632

0 commit comments

Comments
 (0)