Skip to content

Commit eba832e

Browse files
dannyzakendannyzaken
authored
Merge pull request #1660 from dannyzaken/danny-fixes
Added prometheus alert rules for DB volume size capacity
2 parents 7ed7521 + 688af80 commit eba832e

File tree

2 files changed

+125
-1
lines changed

2 files changed

+125
-1
lines changed

‎deploy/internal/prometheus-rules.yaml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,3 +245,65 @@ spec:
245245
for: 5m
246246
labels:
247247
severity: critical
248+
- name: noobaa-db-alert.rules
249+
rules:
250+
- alert: NooBaaDatabaseReachingCapacity
251+
annotations:
252+
description: The NooBaa database on pod noobaa-db-pg-cluster-1 is using 80% of its PVC requested size.
253+
message: NooBaa NooBaa database on pod noobaa-db-pg-cluster-1 is using 80% of its PVC capacity.
254+
severity_level: warning
255+
storage_type: NooBaa
256+
expr: |
257+
(
258+
cnpg_pg_database_size_bytes{datname="nbcore", namespace="openshift-storage", pod="noobaa-db-pg-cluster-1"}
259+
/on(namespace) group_left(persistentvolumeclaim)
260+
kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage", persistentvolumeclaim="noobaa-db-pg-cluster-1"}
261+
) * 100 > 80
262+
for: 5m
263+
labels:
264+
severity: warning
265+
- alert: NooBaaDatabaseReachingCapacity
266+
annotations:
267+
description: The NooBaa database on pod noobaa-db-pg-cluster-2 is using 80% of its PVC requested size.
268+
message: NooBaa NooBaa database on pod noobaa-db-pg-cluster-2 is using 80% of its PVC capacity.
269+
severity_level: warning
270+
storage_type: NooBaa
271+
expr: |
272+
(
273+
cnpg_pg_database_size_bytes{datname="nbcore", namespace="openshift-storage", pod="noobaa-db-pg-cluster-2"}
274+
/on(namespace) group_left(persistentvolumeclaim)
275+
kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage", persistentvolumeclaim="noobaa-db-pg-cluster-2"}
276+
) * 100 > 80
277+
for: 5m
278+
labels:
279+
severity: warning
280+
- alert: NooBaaDatabaseStorageFull
281+
annotations:
282+
description: The NooBaa database on pod noobaa-db-pg-cluster-1 is using over 90% of its PVC requested size. Increase the DB size as soon as possible.
283+
message: NooBaa NooBaa database on pod noobaa-db-pg-cluster-1 is using over 90% of its PVC capacity.
284+
severity_level: critical
285+
storage_type: NooBaa
286+
expr: |
287+
(
288+
cnpg_pg_database_size_bytes{datname="nbcore", namespace="openshift-storage", pod="noobaa-db-pg-cluster-1"}
289+
/on(namespace) group_left(persistentvolumeclaim)
290+
kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage", persistentvolumeclaim="noobaa-db-pg-cluster-1"}
291+
) * 100 > 90
292+
for: 5m
293+
labels:
294+
severity: critical
295+
- alert: NooBaaDatabaseStorageFull
296+
annotations:
297+
description: The NooBaa database on pod noobaa-db-pg-cluster-2 is using over 90% of its PVC requested size. Increase the DB size as soon as possible.
298+
message: NooBaa NooBaa database on pod noobaa-db-pg-cluster-2 is using over 90% of its PVC capacity.
299+
severity_level: critical
300+
storage_type: NooBaa
301+
expr: |
302+
(
303+
cnpg_pg_database_size_bytes{datname="nbcore", namespace="openshift-storage", pod="noobaa-db-pg-cluster-2"}
304+
/on(namespace) group_left(persistentvolumeclaim)
305+
kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage", persistentvolumeclaim="noobaa-db-pg-cluster-2"}
306+
) * 100 > 90
307+
for: 5m
308+
labels:
309+
severity: critical

‎pkg/bundle/deploy.go‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4537,7 +4537,7 @@ spec:
45374537
claimName: noobaa-pv-claim
45384538
`
45394539

4540-
const Sha256_deploy_internal_prometheus_rules_yaml = "a6c6475935673a77c31f3d6bd66b284a5bf6c9b62c05778456795cfee50394ab"
4540+
const Sha256_deploy_internal_prometheus_rules_yaml = "3d136d9c9891c9d3bdfdab4d8b5104ab31329b0740744a2e0bbcb34fa26bebf2"
45414541

45424542
const File_deploy_internal_prometheus_rules_yaml = `apiVersion: monitoring.coreos.com/v1
45434543
kind: PrometheusRule
@@ -4786,6 +4786,68 @@ spec:
47864786
for: 5m
47874787
labels:
47884788
severity: critical
4789+
- name: noobaa-db-alert.rules
4790+
rules:
4791+
- alert: NooBaaDatabaseReachingCapacity
4792+
annotations:
4793+
description: The NooBaa database on pod noobaa-db-pg-cluster-1 is using 80% of its PVC requested size.
4794+
message: NooBaa NooBaa database on pod noobaa-db-pg-cluster-1 is using 80% of its PVC capacity.
4795+
severity_level: warning
4796+
storage_type: NooBaa
4797+
expr: |
4798+
(
4799+
cnpg_pg_database_size_bytes{datname="nbcore", namespace="openshift-storage", pod="noobaa-db-pg-cluster-1"}
4800+
/on(namespace) group_left(persistentvolumeclaim)
4801+
kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage", persistentvolumeclaim="noobaa-db-pg-cluster-1"}
4802+
) * 100 > 80
4803+
for: 5m
4804+
labels:
4805+
severity: warning
4806+
- alert: NooBaaDatabaseReachingCapacity
4807+
annotations:
4808+
description: The NooBaa database on pod noobaa-db-pg-cluster-2 is using 80% of its PVC requested size.
4809+
message: NooBaa NooBaa database on pod noobaa-db-pg-cluster-2 is using 80% of its PVC capacity.
4810+
severity_level: warning
4811+
storage_type: NooBaa
4812+
expr: |
4813+
(
4814+
cnpg_pg_database_size_bytes{datname="nbcore", namespace="openshift-storage", pod="noobaa-db-pg-cluster-2"}
4815+
/on(namespace) group_left(persistentvolumeclaim)
4816+
kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage", persistentvolumeclaim="noobaa-db-pg-cluster-2"}
4817+
) * 100 > 80
4818+
for: 5m
4819+
labels:
4820+
severity: warning
4821+
- alert: NooBaaDatabaseStorageFull
4822+
annotations:
4823+
description: The NooBaa database on pod noobaa-db-pg-cluster-1 is using over 90% of its PVC requested size. Increase the DB size as soon as possible.
4824+
message: NooBaa NooBaa database on pod noobaa-db-pg-cluster-1 is using over 90% of its PVC capacity.
4825+
severity_level: critical
4826+
storage_type: NooBaa
4827+
expr: |
4828+
(
4829+
cnpg_pg_database_size_bytes{datname="nbcore", namespace="openshift-storage", pod="noobaa-db-pg-cluster-1"}
4830+
/on(namespace) group_left(persistentvolumeclaim)
4831+
kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage", persistentvolumeclaim="noobaa-db-pg-cluster-1"}
4832+
) * 100 > 90
4833+
for: 5m
4834+
labels:
4835+
severity: critical
4836+
- alert: NooBaaDatabaseStorageFull
4837+
annotations:
4838+
description: The NooBaa database on pod noobaa-db-pg-cluster-2 is using over 90% of its PVC requested size. Increase the DB size as soon as possible.
4839+
message: NooBaa NooBaa database on pod noobaa-db-pg-cluster-2 is using over 90% of its PVC capacity.
4840+
severity_level: critical
4841+
storage_type: NooBaa
4842+
expr: |
4843+
(
4844+
cnpg_pg_database_size_bytes{datname="nbcore", namespace="openshift-storage", pod="noobaa-db-pg-cluster-2"}
4845+
/on(namespace) group_left(persistentvolumeclaim)
4846+
kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage", persistentvolumeclaim="noobaa-db-pg-cluster-2"}
4847+
) * 100 > 90
4848+
for: 5m
4849+
labels:
4850+
severity: critical
47894851
`
47904852

47914853
const Sha256_deploy_internal_pvc_agent_yaml = "c76fd98867e2e098204377899568a6e1e60062ece903c7bcbeb3444193ec13f8"

0 commit comments

Comments
 (0)