Skip to content

Commit d4a6b38

Browse files
MasterPtatoMasterPtato
committed
feat(gas): implement smart pulling for horizontally scaled workers
1 parent 79bbca9 commit d4a6b38

File tree

25 files changed

+789
-456
lines changed

25 files changed

+789
-456
lines changed

‎Cargo.lock‎

Lines changed: 43 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ slog = "2.7"
6969
slog-async = "2.8"
7070
slog-term = "2.9"
7171
statrs = "0.18"
72+
sysinfo = "0.37.2"
7273
tabled = "0.17.0"
7374
tempfile = "3.13.0"
7475
testcontainers = "0.24"

‎engine/docker/dev-host/grafana/dashboards/gasoline.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1087,12 +1087,12 @@
10871087
"editorType": "sql",
10881088
"format": 1,
10891089
"instant": false,
1090-
"legendFormat": "{{worker_instance_id}}",
1090+
"legendFormat": "{{worker_id}}",
10911091
"meta": {},
10921092
"pluginVersion": "4.11.2",
10931093
"queryType": "table",
10941094
"range": true,
1095-
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
1095+
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
10961096
"refId": "A"
10971097
}
10981098
],
@@ -1211,12 +1211,12 @@
12111211
"editorType": "sql",
12121212
"format": 1,
12131213
"instant": false,
1214-
"legendFormat": "{{worker_instance_id}}",
1214+
"legendFormat": "{{worker_id}}",
12151215
"meta": {},
12161216
"pluginVersion": "4.11.2",
12171217
"queryType": "table",
12181218
"range": true,
1219-
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
1219+
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
12201220
"refId": "A"
12211221
}
12221222
],

‎engine/docker/dev-multidc-multinode/core/grafana/dashboards/gasoline.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1087,12 +1087,12 @@
10871087
"editorType": "sql",
10881088
"format": 1,
10891089
"instant": false,
1090-
"legendFormat": "{{worker_instance_id}}",
1090+
"legendFormat": "{{worker_id}}",
10911091
"meta": {},
10921092
"pluginVersion": "4.11.2",
10931093
"queryType": "table",
10941094
"range": true,
1095-
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
1095+
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
10961096
"refId": "A"
10971097
}
10981098
],
@@ -1211,12 +1211,12 @@
12111211
"editorType": "sql",
12121212
"format": 1,
12131213
"instant": false,
1214-
"legendFormat": "{{worker_instance_id}}",
1214+
"legendFormat": "{{worker_id}}",
12151215
"meta": {},
12161216
"pluginVersion": "4.11.2",
12171217
"queryType": "table",
12181218
"range": true,
1219-
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
1219+
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
12201220
"refId": "A"
12211221
}
12221222
],

‎engine/docker/dev-multidc/core/grafana/dashboards/gasoline.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1087,12 +1087,12 @@
10871087
"editorType": "sql",
10881088
"format": 1,
10891089
"instant": false,
1090-
"legendFormat": "{{worker_instance_id}}",
1090+
"legendFormat": "{{worker_id}}",
10911091
"meta": {},
10921092
"pluginVersion": "4.11.2",
10931093
"queryType": "table",
10941094
"range": true,
1095-
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
1095+
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
10961096
"refId": "A"
10971097
}
10981098
],
@@ -1211,12 +1211,12 @@
12111211
"editorType": "sql",
12121212
"format": 1,
12131213
"instant": false,
1214-
"legendFormat": "{{worker_instance_id}}",
1214+
"legendFormat": "{{worker_id}}",
12151215
"meta": {},
12161216
"pluginVersion": "4.11.2",
12171217
"queryType": "table",
12181218
"range": true,
1219-
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
1219+
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
12201220
"refId": "A"
12211221
}
12221222
],

‎engine/docker/dev-multinode/grafana/dashboards/gasoline.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1087,12 +1087,12 @@
10871087
"editorType": "sql",
10881088
"format": 1,
10891089
"instant": false,
1090-
"legendFormat": "{{worker_instance_id}}",
1090+
"legendFormat": "{{worker_id}}",
10911091
"meta": {},
10921092
"pluginVersion": "4.11.2",
10931093
"queryType": "table",
10941094
"range": true,
1095-
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
1095+
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
10961096
"refId": "A"
10971097
}
10981098
],
@@ -1211,12 +1211,12 @@
12111211
"editorType": "sql",
12121212
"format": 1,
12131213
"instant": false,
1214-
"legendFormat": "{{worker_instance_id}}",
1214+
"legendFormat": "{{worker_id}}",
12151215
"meta": {},
12161216
"pluginVersion": "4.11.2",
12171217
"queryType": "table",
12181218
"range": true,
1219-
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
1219+
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
12201220
"refId": "A"
12211221
}
12221222
],

‎engine/docker/dev/grafana/dashboards/gasoline.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1087,12 +1087,12 @@
10871087
"editorType": "sql",
10881088
"format": 1,
10891089
"instant": false,
1090-
"legendFormat": "{{worker_instance_id}}",
1090+
"legendFormat": "{{worker_id}}",
10911091
"meta": {},
10921092
"pluginVersion": "4.11.2",
10931093
"queryType": "table",
10941094
"range": true,
1095-
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
1095+
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
10961096
"refId": "A"
10971097
}
10981098
],
@@ -1211,12 +1211,12 @@
12111211
"editorType": "sql",
12121212
"format": 1,
12131213
"instant": false,
1214-
"legendFormat": "{{worker_instance_id}}",
1214+
"legendFormat": "{{worker_id}}",
12151215
"meta": {},
12161216
"pluginVersion": "4.11.2",
12171217
"queryType": "table",
12181218
"range": true,
1219-
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
1219+
"rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label",
12201220
"refId": "A"
12211221
}
12221222
],

0 commit comments

Comments
 (0)