Skip to content

Commit 7408562

Browse files
iambriccardoiambriccardo
authored
feat(k8s): Add topology constraints (#441)
1 parent 7b71d80 commit 7408562

7 files changed

+178
-25
lines changed

‎etl-api/src/k8s/http.rs‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,6 @@ const LOGS_VOLUME_NAME: &str = "logs";
6565
pub const TRUSTED_ROOT_CERT_CONFIG_MAP_NAME: &str = "trusted-root-certs-config";
6666
/// Key inside the trusted root certificates ConfigMap.
6767
pub const TRUSTED_ROOT_CERT_KEY_NAME: &str = "trusted_root_certs";
68-
/// Pod template annotation used to trigger rolling restarts.
69-
const RESTARTED_AT_ANNOTATION_KEY: &str = "etl.supabase.com/restarted-at";
7068
/// Label used to identify replicator pods.
7169
const REPLICATOR_APP_LABEL: &str = "etl-replicator-app";
7270

@@ -663,7 +661,7 @@ fn create_container_environment_json(
663661
}
664662

665663
fn create_node_selector_json(environment: &Environment) -> serde_json::Value {
666-
// In staging and prod, pin pods to nodes labeled with `nodeType=workloads`.
664+
// In staging and prod, pin pods to workload pods.
667665
match environment {
668666
Environment::Dev => json!({}),
669667
Environment::Staging | Environment::Prod => json!({
@@ -867,18 +865,18 @@ fn create_replicator_stateful_set_json(
867865
"replicas": 1,
868866
"selector": {
869867
"matchLabels": {
870-
"app-name": replicator_app_name,
868+
"etl.supabase.com/app-name": replicator_app_name,
871869
}
872870
},
873871
"template": {
874872
"metadata": {
875873
"labels": {
876-
"app-name": replicator_app_name,
877-
"app": REPLICATOR_APP_LABEL
874+
"etl.supabase.com/app-name": replicator_app_name,
875+
"etl.supabase.com/app-type": REPLICATOR_APP_LABEL
878876
},
879877
"annotations": {
880878
// Attach template annotations (e.g., restart checksum) to trigger a rolling restart
881-
RESTARTED_AT_ANNOTATION_KEY: restarted_at_annotation,
879+
"etl.supabase.com/restarted-at": restarted_at_annotation,
882880
}
883881
},
884882
"spec": {
@@ -892,6 +890,29 @@ fn create_replicator_stateful_set_json(
892890
"effect": "NoSchedule"
893891
}
894892
],
893+
// Distribute pods evenly across nodes and availability zones.
894+
"topologySpreadConstraints": [
895+
{
896+
"maxSkew": 1,
897+
"topologyKey": "kubernetes.io/hostname",
898+
"whenUnsatisfiable": "ScheduleAnyway",
899+
"labelSelector": {
900+
"matchLabels": {
901+
"etl.supabase.com/app-type": REPLICATOR_APP_LABEL
902+
}
903+
}
904+
},
905+
{
906+
"maxSkew": 1,
907+
"topologyKey": "topology.kubernetes.io/zone",
908+
"whenUnsatisfiable": "ScheduleAnyway",
909+
"labelSelector": {
910+
"matchLabels": {
911+
"etl.supabase.com/app-type": REPLICATOR_APP_LABEL
912+
}
913+
}
914+
}
915+
],
895916
"nodeSelector": node_selector,
896917
// We want to wait at most 5 minutes before K8S sends a `SIGKILL` to the containers,
897918
// this way we let the system finish any in-flight transaction, if there are any.

‎etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_replicator_stateful_set_json-2.snap‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ expression: stateful_set_json
1313
"replicas": 1,
1414
"selector": {
1515
"matchLabels": {
16-
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
16+
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app"
1717
}
1818
},
1919
"template": {
@@ -22,8 +22,8 @@ expression: stateful_set_json
2222
"etl.supabase.com/restarted-at": "[timestamp]"
2323
},
2424
"labels": {
25-
"app": "etl-replicator-app",
26-
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
25+
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app",
26+
"etl.supabase.com/app-type": "etl-replicator-app"
2727
}
2828
},
2929
"spec": {
@@ -162,6 +162,28 @@ expression: stateful_set_json
162162
"value": "workloads"
163163
}
164164
],
165+
"topologySpreadConstraints": [
166+
{
167+
"labelSelector": {
168+
"matchLabels": {
169+
"etl.supabase.com/app-type": "etl-replicator-app"
170+
}
171+
},
172+
"maxSkew": 1,
173+
"topologyKey": "kubernetes.io/hostname",
174+
"whenUnsatisfiable": "ScheduleAnyway"
175+
},
176+
{
177+
"labelSelector": {
178+
"matchLabels": {
179+
"etl.supabase.com/app-type": "etl-replicator-app"
180+
}
181+
},
182+
"maxSkew": 1,
183+
"topologyKey": "topology.kubernetes.io/zone",
184+
"whenUnsatisfiable": "ScheduleAnyway"
185+
}
186+
],
165187
"volumes": [
166188
{
167189
"configMap": {

‎etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_replicator_stateful_set_json-3.snap‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ expression: stateful_set_json
1313
"replicas": 1,
1414
"selector": {
1515
"matchLabels": {
16-
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
16+
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app"
1717
}
1818
},
1919
"template": {
@@ -22,8 +22,8 @@ expression: stateful_set_json
2222
"etl.supabase.com/restarted-at": "[timestamp]"
2323
},
2424
"labels": {
25-
"app": "etl-replicator-app",
26-
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
25+
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app",
26+
"etl.supabase.com/app-type": "etl-replicator-app"
2727
}
2828
},
2929
"spec": {
@@ -162,6 +162,28 @@ expression: stateful_set_json
162162
"value": "workloads"
163163
}
164164
],
165+
"topologySpreadConstraints": [
166+
{
167+
"labelSelector": {
168+
"matchLabels": {
169+
"etl.supabase.com/app-type": "etl-replicator-app"
170+
}
171+
},
172+
"maxSkew": 1,
173+
"topologyKey": "kubernetes.io/hostname",
174+
"whenUnsatisfiable": "ScheduleAnyway"
175+
},
176+
{
177+
"labelSelector": {
178+
"matchLabels": {
179+
"etl.supabase.com/app-type": "etl-replicator-app"
180+
}
181+
},
182+
"maxSkew": 1,
183+
"topologyKey": "topology.kubernetes.io/zone",
184+
"whenUnsatisfiable": "ScheduleAnyway"
185+
}
186+
],
165187
"volumes": [
166188
{
167189
"configMap": {

‎etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_replicator_stateful_set_json.snap‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ expression: stateful_set_json
1313
"replicas": 1,
1414
"selector": {
1515
"matchLabels": {
16-
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
16+
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app"
1717
}
1818
},
1919
"template": {
@@ -22,8 +22,8 @@ expression: stateful_set_json
2222
"etl.supabase.com/restarted-at": "[timestamp]"
2323
},
2424
"labels": {
25-
"app": "etl-replicator-app",
26-
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
25+
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app",
26+
"etl.supabase.com/app-type": "etl-replicator-app"
2727
}
2828
},
2929
"spec": {
@@ -89,6 +89,28 @@ expression: stateful_set_json
8989
"value": "workloads"
9090
}
9191
],
92+
"topologySpreadConstraints": [
93+
{
94+
"labelSelector": {
95+
"matchLabels": {
96+
"etl.supabase.com/app-type": "etl-replicator-app"
97+
}
98+
},
99+
"maxSkew": 1,
100+
"topologyKey": "kubernetes.io/hostname",
101+
"whenUnsatisfiable": "ScheduleAnyway"
102+
},
103+
{
104+
"labelSelector": {
105+
"matchLabels": {
106+
"etl.supabase.com/app-type": "etl-replicator-app"
107+
}
108+
},
109+
"maxSkew": 1,
110+
"topologyKey": "topology.kubernetes.io/zone",
111+
"whenUnsatisfiable": "ScheduleAnyway"
112+
}
113+
],
92114
"volumes": [
93115
{
94116
"configMap": {

‎etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_replicator_stateful_set_json-2.snap‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ expression: stateful_set_json
1313
"replicas": 1,
1414
"selector": {
1515
"matchLabels": {
16-
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
16+
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app"
1717
}
1818
},
1919
"template": {
@@ -22,8 +22,8 @@ expression: stateful_set_json
2222
"etl.supabase.com/restarted-at": "[timestamp]"
2323
},
2424
"labels": {
25-
"app": "etl-replicator-app",
26-
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
25+
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app",
26+
"etl.supabase.com/app-type": "etl-replicator-app"
2727
}
2828
},
2929
"spec": {
@@ -180,6 +180,28 @@ expression: stateful_set_json
180180
"value": "workloads"
181181
}
182182
],
183+
"topologySpreadConstraints": [
184+
{
185+
"labelSelector": {
186+
"matchLabels": {
187+
"etl.supabase.com/app-type": "etl-replicator-app"
188+
}
189+
},
190+
"maxSkew": 1,
191+
"topologyKey": "kubernetes.io/hostname",
192+
"whenUnsatisfiable": "ScheduleAnyway"
193+
},
194+
{
195+
"labelSelector": {
196+
"matchLabels": {
197+
"etl.supabase.com/app-type": "etl-replicator-app"
198+
}
199+
},
200+
"maxSkew": 1,
201+
"topologyKey": "topology.kubernetes.io/zone",
202+
"whenUnsatisfiable": "ScheduleAnyway"
203+
}
204+
],
183205
"volumes": [
184206
{
185207
"configMap": {

‎etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_replicator_stateful_set_json-3.snap‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ expression: stateful_set_json
1313
"replicas": 1,
1414
"selector": {
1515
"matchLabels": {
16-
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
16+
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app"
1717
}
1818
},
1919
"template": {
@@ -22,8 +22,8 @@ expression: stateful_set_json
2222
"etl.supabase.com/restarted-at": "[timestamp]"
2323
},
2424
"labels": {
25-
"app": "etl-replicator-app",
26-
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
25+
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app",
26+
"etl.supabase.com/app-type": "etl-replicator-app"
2727
}
2828
},
2929
"spec": {
@@ -180,6 +180,28 @@ expression: stateful_set_json
180180
"value": "workloads"
181181
}
182182
],
183+
"topologySpreadConstraints": [
184+
{
185+
"labelSelector": {
186+
"matchLabels": {
187+
"etl.supabase.com/app-type": "etl-replicator-app"
188+
}
189+
},
190+
"maxSkew": 1,
191+
"topologyKey": "kubernetes.io/hostname",
192+
"whenUnsatisfiable": "ScheduleAnyway"
193+
},
194+
{
195+
"labelSelector": {
196+
"matchLabels": {
197+
"etl.supabase.com/app-type": "etl-replicator-app"
198+
}
199+
},
200+
"maxSkew": 1,
201+
"topologyKey": "topology.kubernetes.io/zone",
202+
"whenUnsatisfiable": "ScheduleAnyway"
203+
}
204+
],
183205
"volumes": [
184206
{
185207
"configMap": {

‎etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_replicator_stateful_set_json.snap‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ expression: stateful_set_json
1313
"replicas": 1,
1414
"selector": {
1515
"matchLabels": {
16-
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
16+
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app"
1717
}
1818
},
1919
"template": {
@@ -22,8 +22,8 @@ expression: stateful_set_json
2222
"etl.supabase.com/restarted-at": "[timestamp]"
2323
},
2424
"labels": {
25-
"app": "etl-replicator-app",
26-
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
25+
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app",
26+
"etl.supabase.com/app-type": "etl-replicator-app"
2727
}
2828
},
2929
"spec": {
@@ -107,6 +107,28 @@ expression: stateful_set_json
107107
"value": "workloads"
108108
}
109109
],
110+
"topologySpreadConstraints": [
111+
{
112+
"labelSelector": {
113+
"matchLabels": {
114+
"etl.supabase.com/app-type": "etl-replicator-app"
115+
}
116+
},
117+
"maxSkew": 1,
118+
"topologyKey": "kubernetes.io/hostname",
119+
"whenUnsatisfiable": "ScheduleAnyway"
120+
},
121+
{
122+
"labelSelector": {
123+
"matchLabels": {
124+
"etl.supabase.com/app-type": "etl-replicator-app"
125+
}
126+
},
127+
"maxSkew": 1,
128+
"topologyKey": "topology.kubernetes.io/zone",
129+
"whenUnsatisfiable": "ScheduleAnyway"
130+
}
131+
],
110132
"volumes": [
111133
{
112134
"configMap": {

0 commit comments

Comments
 (0)