Skip to content

feat(k8s): Add topology constraints #441

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 10, 2025
Merged

feat(k8s): Add topology constraints #441

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c99fcf9
feat(k8s): Add topology constraints
iambriccardo Nov 7, 2025
f2814fd
Improve
iambriccardo Nov 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 28 additions & 7 deletions etl-api/src/k8s/http.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,6 @@ const LOGS_VOLUME_NAME: &str = "logs";
pub const TRUSTED_ROOT_CERT_CONFIG_MAP_NAME: &str = "trusted-root-certs-config";
/// Key inside the trusted root certificates ConfigMap.
pub const TRUSTED_ROOT_CERT_KEY_NAME: &str = "trusted_root_certs";
/// Pod template annotation used to trigger rolling restarts.
const RESTARTED_AT_ANNOTATION_KEY: &str = "etl.supabase.com/restarted-at";
/// Label used to identify replicator pods.
const REPLICATOR_APP_LABEL: &str = "etl-replicator-app";

Expand Down Expand Up @@ -663,7 +661,7 @@ fn create_container_environment_json(
}

fn create_node_selector_json(environment: &Environment) -> serde_json::Value {
// In staging and prod, pin pods to nodes labeled with `nodeType=workloads`.
// In staging and prod, pin pods to workload pods.
match environment {
Environment::Dev => json!({}),
Environment::Staging | Environment::Prod => json!({
Expand Down Expand Up @@ -867,18 +865,18 @@ fn create_replicator_stateful_set_json(
"replicas": 1,
"selector": {
"matchLabels": {
"app-name": replicator_app_name,
"etl.supabase.com/app-name": replicator_app_name,
}
},
"template": {
"metadata": {
"labels": {
"app-name": replicator_app_name,
"app": REPLICATOR_APP_LABEL
"etl.supabase.com/app-name": replicator_app_name,
"etl.supabase.com/app-type": REPLICATOR_APP_LABEL
},
"annotations": {
// Attach template annotations (e.g., restart checksum) to trigger a rolling restart
RESTARTED_AT_ANNOTATION_KEY: restarted_at_annotation,
"etl.supabase.com/restarted-at": restarted_at_annotation,
}
},
"spec": {
Expand All @@ -892,6 +890,29 @@ fn create_replicator_stateful_set_json(
"effect": "NoSchedule"
}
],
// Distribute pods evenly across nodes and availability zones.
"topologySpreadConstraints": [
{
"maxSkew": 1,
"topologyKey": "kubernetes.io/hostname",
"whenUnsatisfiable": "ScheduleAnyway",
"labelSelector": {
"matchLabels": {
"etl.supabase.com/app-type": REPLICATOR_APP_LABEL
}
}
},
{
"maxSkew": 1,
"topologyKey": "topology.kubernetes.io/zone",
"whenUnsatisfiable": "ScheduleAnyway",
"labelSelector": {
"matchLabels": {
"etl.supabase.com/app-type": REPLICATOR_APP_LABEL
}
}
}
],
"nodeSelector": node_selector,
// We want to wait at most 5 minutes before K8S sends a `SIGKILL` to the containers,
// this way we let the system finish any in-flight transaction, if there are any.
Expand Down
28 changes: 25 additions & 3 deletions ...rc/k8s/snapshots/etl_api__k8s__http__tests__create_bq_replicator_stateful_set_json-2.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ expression: stateful_set_json
"replicas": 1,
"selector": {
"matchLabels": {
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app"
}
},
"template": {
Expand All @@ -22,8 +22,8 @@ expression: stateful_set_json
"etl.supabase.com/restarted-at": "[timestamp]"
},
"labels": {
"app": "etl-replicator-app",
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app",
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"spec": {
Expand Down Expand Up @@ -162,6 +162,28 @@ expression: stateful_set_json
"value": "workloads"
}
],
"topologySpreadConstraints": [
{
"labelSelector": {
"matchLabels": {
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"maxSkew": 1,
"topologyKey": "kubernetes.io/hostname",
"whenUnsatisfiable": "ScheduleAnyway"
},
{
"labelSelector": {
"matchLabels": {
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"maxSkew": 1,
"topologyKey": "topology.kubernetes.io/zone",
"whenUnsatisfiable": "ScheduleAnyway"
}
],
"volumes": [
{
"configMap": {
Expand Down
28 changes: 25 additions & 3 deletions ...rc/k8s/snapshots/etl_api__k8s__http__tests__create_bq_replicator_stateful_set_json-3.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ expression: stateful_set_json
"replicas": 1,
"selector": {
"matchLabels": {
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app"
}
},
"template": {
Expand All @@ -22,8 +22,8 @@ expression: stateful_set_json
"etl.supabase.com/restarted-at": "[timestamp]"
},
"labels": {
"app": "etl-replicator-app",
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app",
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"spec": {
Expand Down Expand Up @@ -162,6 +162,28 @@ expression: stateful_set_json
"value": "workloads"
}
],
"topologySpreadConstraints": [
{
"labelSelector": {
"matchLabels": {
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"maxSkew": 1,
"topologyKey": "kubernetes.io/hostname",
"whenUnsatisfiable": "ScheduleAnyway"
},
{
"labelSelector": {
"matchLabels": {
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"maxSkew": 1,
"topologyKey": "topology.kubernetes.io/zone",
"whenUnsatisfiable": "ScheduleAnyway"
}
],
"volumes": [
{
"configMap": {
Expand Down
28 changes: 25 additions & 3 deletions .../src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_replicator_stateful_set_json.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ expression: stateful_set_json
"replicas": 1,
"selector": {
"matchLabels": {
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app"
}
},
"template": {
Expand All @@ -22,8 +22,8 @@ expression: stateful_set_json
"etl.supabase.com/restarted-at": "[timestamp]"
},
"labels": {
"app": "etl-replicator-app",
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app",
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"spec": {
Expand Down Expand Up @@ -89,6 +89,28 @@ expression: stateful_set_json
"value": "workloads"
}
],
"topologySpreadConstraints": [
{
"labelSelector": {
"matchLabels": {
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"maxSkew": 1,
"topologyKey": "kubernetes.io/hostname",
"whenUnsatisfiable": "ScheduleAnyway"
},
{
"labelSelector": {
"matchLabels": {
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"maxSkew": 1,
"topologyKey": "topology.kubernetes.io/zone",
"whenUnsatisfiable": "ScheduleAnyway"
}
],
"volumes": [
{
"configMap": {
Expand Down
28 changes: 25 additions & 3 deletions ...s/snapshots/etl_api__k8s__http__tests__create_iceberg_replicator_stateful_set_json-2.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ expression: stateful_set_json
"replicas": 1,
"selector": {
"matchLabels": {
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app"
}
},
"template": {
Expand All @@ -22,8 +22,8 @@ expression: stateful_set_json
"etl.supabase.com/restarted-at": "[timestamp]"
},
"labels": {
"app": "etl-replicator-app",
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app",
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"spec": {
Expand Down Expand Up @@ -180,6 +180,28 @@ expression: stateful_set_json
"value": "workloads"
}
],
"topologySpreadConstraints": [
{
"labelSelector": {
"matchLabels": {
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"maxSkew": 1,
"topologyKey": "kubernetes.io/hostname",
"whenUnsatisfiable": "ScheduleAnyway"
},
{
"labelSelector": {
"matchLabels": {
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"maxSkew": 1,
"topologyKey": "topology.kubernetes.io/zone",
"whenUnsatisfiable": "ScheduleAnyway"
}
],
"volumes": [
{
"configMap": {
Expand Down
28 changes: 25 additions & 3 deletions ...s/snapshots/etl_api__k8s__http__tests__create_iceberg_replicator_stateful_set_json-3.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ expression: stateful_set_json
"replicas": 1,
"selector": {
"matchLabels": {
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app"
}
},
"template": {
Expand All @@ -22,8 +22,8 @@ expression: stateful_set_json
"etl.supabase.com/restarted-at": "[timestamp]"
},
"labels": {
"app": "etl-replicator-app",
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app",
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"spec": {
Expand Down Expand Up @@ -180,6 +180,28 @@ expression: stateful_set_json
"value": "workloads"
}
],
"topologySpreadConstraints": [
{
"labelSelector": {
"matchLabels": {
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"maxSkew": 1,
"topologyKey": "kubernetes.io/hostname",
"whenUnsatisfiable": "ScheduleAnyway"
},
{
"labelSelector": {
"matchLabels": {
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"maxSkew": 1,
"topologyKey": "topology.kubernetes.io/zone",
"whenUnsatisfiable": "ScheduleAnyway"
}
],
"volumes": [
{
"configMap": {
Expand Down
28 changes: 25 additions & 3 deletions ...k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_replicator_stateful_set_json.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ expression: stateful_set_json
"replicas": 1,
"selector": {
"matchLabels": {
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app"
}
},
"template": {
Expand All @@ -22,8 +22,8 @@ expression: stateful_set_json
"etl.supabase.com/restarted-at": "[timestamp]"
},
"labels": {
"app": "etl-replicator-app",
"app-name": "abcdefghijklmnopqrst-42-replicator-app"
"etl.supabase.com/app-name": "abcdefghijklmnopqrst-42-replicator-app",
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"spec": {
Expand Down Expand Up @@ -107,6 +107,28 @@ expression: stateful_set_json
"value": "workloads"
}
],
"topologySpreadConstraints": [
{
"labelSelector": {
"matchLabels": {
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"maxSkew": 1,
"topologyKey": "kubernetes.io/hostname",
"whenUnsatisfiable": "ScheduleAnyway"
},
{
"labelSelector": {
"matchLabels": {
"etl.supabase.com/app-type": "etl-replicator-app"
}
},
"maxSkew": 1,
"topologyKey": "topology.kubernetes.io/zone",
"whenUnsatisfiable": "ScheduleAnyway"
}
],
"volumes": [
{
"configMap": {
Expand Down
Loading