diff --git a/cardano-db-sync/app/cardano-db-sync.hs b/cardano-db-sync/app/cardano-db-sync.hs index d708fff2b..3f070f07d 100644 --- a/cardano-db-sync/app/cardano-db-sync.hs +++ b/cardano-db-sync/app/cardano-db-sync.hs @@ -1,8 +1,13 @@ +{-# LANGUAGE CPP #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE NoImplicitPrelude #-} import Cardano.Db (MigrationDir (..), PGPassSource (PGPassDefaultEnv, PGPassEnv), gitRev) import Cardano.DbSync (runDbSyncNode) + +#ifdef GHC_DEBUG_ENABLED +import GHC.Debug.Stub (withGhcDebug) +#endif import Cardano.DbSync.Config import Cardano.DbSync.Config.Types import Cardano.DbSync.Metrics (withMetricSetters) @@ -21,8 +26,16 @@ import Prelude (error) --------------------------------------------------------------------------------------------------- -- Main entry point into the app --------------------------------------------------------------------------------------------------- + main :: IO () -main = do +#ifdef GHC_DEBUG_ENABLED +main = withGhcDebug dbSyncMain +#else +main = dbSyncMain +#endif + +dbSyncMain :: IO () +dbSyncMain = do cmd <- Opt.execParser opts case cmd of CmdVersion -> runVersionCommand diff --git a/cardano-db-sync/cardano-db-sync.cabal b/cardano-db-sync/cardano-db-sync.cabal index d3e136fa6..4bbc16a24 100644 --- a/cardano-db-sync/cardano-db-sync.cabal +++ b/cardano-db-sync/cardano-db-sync.cabal @@ -17,6 +17,13 @@ build-type: Custom extra-source-files: CHANGELOG.md schema/*.sql +flag enable-ghc-debug + description: Enable ghc-debug profiling support for heap analysis. + Build with: cabal build --flag enable-ghc-debug cardano-db-sync + Zero overhead when disabled (production builds). + default: False + manual: True + custom-setup setup-depends: base @@ -252,6 +259,12 @@ executable cardano-db-sync , optparse-applicative , text + if flag(enable-ghc-debug) + build-depends: ghc-debug-stub + cpp-options: -DGHC_DEBUG_ENABLED + ghc-options: -finfo-table-map + -fdistinct-constructor-tables + executable http-get-json-metadata default-language: Haskell2010 main-is: http-get-json-metadata.hs diff --git a/dev-tools/README.md b/dev-tools/README.md new file mode 100644 index 000000000..4bdec8f24 --- /dev/null +++ b/dev-tools/README.md @@ -0,0 +1,110 @@ +# Cardano DB Sync - Developer Tools + +Local development tools for monitoring and profiling cardano-db-sync. + +## Overview + +Two complementary tools for understanding cardano-db-sync performance: + +### πŸ” [Monitoring](monitoring/README.md) +Real-time metrics with Prometheus/Grafana +- PostgreSQL metrics (queries, cache, connections) +- System metrics (CPU, memory, disk I/O) +- Time-series visualization + +### 🧠 [Profiling](profiling/README.md) +Memory profiling with ghc-debug +- Interactive heap exploration +- Memory leak detection +- Retainer chain analysis + +## Installation + +### Monitoring + +```bash +# macOS +brew install tmux prometheus postgres_exporter node_exporter grafana + +# Linux (apt) +sudo apt-get install tmux prometheus postgres-exporter prometheus-node-exporter grafana + +# Linux (yum) +sudo yum install tmux prometheus postgres_exporter node_exporter grafana +``` + +### Profiling + +```bash +# Install ghc-debug-brick +git clone https://gitlab.haskell.org/ghc/ghc-debug.git +cd ghc-debug/brick +cabal install ghc-debug-brick + +# Instrument cardano-db-sync (see profiling/README.md for details) +``` + +## Running + +### Start Monitoring + +```bash +cd dev-tools/monitoring +./scripts/start-monitoring.sh +``` + +Access at: +- Prometheus: http://localhost:9090 +- Grafana: http://localhost:3000 (if started separately) + +### Start Profiling + +```bash +cd dev-tools/profiling +./scripts/start-profiling.sh + +# In another terminal (after several hours): +ghc-debug-brick /tmp/cardano-db-sync.ghc-debug +``` + +### Run Both Together + +```bash +# Terminal 1: Monitoring +cd dev-tools/monitoring && ./scripts/start-monitoring.sh + +# Terminal 2: Profiling +cd dev-tools/profiling && ./scripts/start-profiling.sh +``` + +## Directory Structure + +``` +dev-tools/ +β”œβ”€β”€ README.md # This file +β”‚ +β”œβ”€β”€ monitoring/ # Prometheus/Grafana monitoring +β”‚ β”œβ”€β”€ README.md # Full monitoring documentation +β”‚ β”œβ”€β”€ scripts/ +β”‚ β”‚ └── start-monitoring.sh +β”‚ β”œβ”€β”€ config/ +β”‚ β”‚ └── prometheus.yml +β”‚ β”œβ”€β”€ data/ # Prometheus data (gitignored) +β”‚ └── docs/ +β”‚ └── METRICS.md # Available metrics +β”‚ +└── profiling/ # ghc-debug profiling + β”œβ”€β”€ README.md # Full profiling documentation + β”œβ”€β”€ scripts/ + β”‚ β”œβ”€β”€ start-profiling.sh + β”‚ └── analyze-memory.sh + β”œβ”€β”€ snapshots/ # Heap snapshots (gitignored) + β”œβ”€β”€ logs/ # Memory logs (gitignored) + └── reports/ # Analysis reports (commit these) +``` + +## Documentation + +- [Monitoring Setup Guide](monitoring/README.md) +- [Profiling Setup Guide](profiling/README.md) +- [Available Metrics Reference](monitoring/docs/METRICS.md) diff --git a/dev-tools/monitoring/README.md b/dev-tools/monitoring/README.md new file mode 100644 index 000000000..836962b1c --- /dev/null +++ b/dev-tools/monitoring/README.md @@ -0,0 +1,114 @@ +# Cardano DB Sync - Monitoring + +Local development monitoring using Prometheus and Grafana for PostgreSQL and system metrics. + +## Quick Setup + +### 1. Install Tools + +```bash +# macOS +brew install prometheus postgres_exporter node_exporter grafana + +# Linux (apt) +sudo apt-get install prometheus postgres-exporter prometheus-node-exporter grafana + +# Linux (yum) +sudo yum install prometheus postgres_exporter node_exporter grafana +``` + +### 2. Setup PostgreSQL Monitoring User + +```sql +-- Connect to your database +psql -U postgres -d cexplorer + +-- Create monitoring user +CREATE USER postgres_exporter WITH PASSWORD 'secure_password'; +GRANT pg_monitor TO postgres_exporter; +GRANT CONNECT ON DATABASE cexplorer TO postgres_exporter; +``` + +### 3. Start Monitoring + +```bash +cd dev-tools/monitoring +./scripts/start-monitoring.sh +``` + +This launches a tmux session with Prometheus and exporters. + +### 4. Start Grafana + +```bash +# macOS +brew services start grafana + +# Linux +sudo systemctl start grafana-server +``` + +## Access Dashboards + +- **Prometheus**: http://localhost:9090 +- **Grafana**: http://localhost:3000 (default: admin/admin) + +## Grafana Setup + +1. Open http://localhost:3000 +2. Add Prometheus data source: + - Configuration β†’ Data Sources β†’ Add data source + - Select Prometheus + - URL: `http://localhost:9090` + - Save & Test +3. Import dashboard: + - Dashboards β†’ Import + - Upload `config/grafana-db-sync.json` + - Select Prometheus data source + +## Key Metrics + +- **System**: CPU, memory usage (node_exporter) +- **PostgreSQL**: Connections, transaction rates, cache hit ratio +- **Sync Progress**: Block height, epoch duration + +## Troubleshooting + +**postgres_exporter fails to connect**: +- Verify monitoring user exists and has permissions +- Set `DATA_SOURCE_NAME` with correct credentials in `scripts/start-monitoring.sh` + +**Prometheus shows "DOWN" targets**: +- Check exporters are running: `ps aux | grep exporter` +- Test endpoints: `curl http://localhost:9187/metrics` and `curl http://localhost:9100/metrics` + +**Port already in use**: +```bash +# Kill processes using ports +lsof -ti:9090 | xargs kill # Prometheus +lsof -ti:9100 | xargs kill # node_exporter +lsof -ti:9187 | xargs kill # postgres_exporter +``` + +## Stop Services + +```bash +# Stop monitoring +tmux kill-session -t cardano-monitoring + +# Stop Grafana +brew services stop grafana # macOS +sudo systemctl stop grafana-server # Linux +``` + +## Files + +- `scripts/start-monitoring.sh` - Launch monitoring suite +- `config/prometheus.yml` - Prometheus configuration +- `config/grafana-db-sync.json` - Grafana dashboard +- `docs/METRICS.md` - Complete metrics reference + +## See Also + +- [Prometheus Documentation](https://prometheus.io/docs/) +- [Grafana Documentation](https://grafana.com/docs/) diff --git a/dev-tools/monitoring/config/grafana-db-sync.json b/dev-tools/monitoring/config/grafana-db-sync.json new file mode 100644 index 000000000..19f26fa0e --- /dev/null +++ b/dev-tools/monitoring/config/grafana-db-sync.json @@ -0,0 +1,3247 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "This dashboard works with postgres_exporter for prometheus", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 10, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 34, + "panels": [], + "title": "General Counters, CPU, Memory and File Descriptor Stats", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 36, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_static{release=\"$release\", instance=\"$instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{short_version}}", + "refId": "A" + } + ], + "title": "Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "description": "start time of the process", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dateTimeFromNow" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 28, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_postmaster_start_time_seconds{release=\"$release\", instance=\"$instance\"} * 1000", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Start Time", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 10, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "SUM(pg_stat_database_tup_fetched{datname=~\"$datname\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "title": "Current fetch data", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 11, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "SUM(pg_stat_database_tup_inserted{release=\"$release\", datname=~\"$datname\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "title": "Current insert data", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 12, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "SUM(pg_stat_database_tup_updated{datname=~\"$datname\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "title": "Current update data", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 38, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_settings_max_connections{release=\"$release\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Max Connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "timezone": [ + "browser" + ], + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(\n (1 - rate(node_cpu_seconds_total{job=\"node\", mode=\"idle\", instance=\"$node\"}[$__interval]))\n/ ignoring(cpu) group_left\n count without (cpu)( node_cpu_seconds_total{job=\"node\", mode=\"idle\", instance=\"$node\"})\n)\n", + "format": "time_series", + "interval": "1m", + "intervalFactor": 5, + "legendFormat": "{{cpu}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "memory used, not incl. wired" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(\n node_memory_total_bytes{job=\"node\", instance=\"$node\"}\n-\n node_memory_free_bytes{job=\"node\", instance=\"$node\"}\n-\n node_memory_wired_bytes{job=\"node\", instance=\"$node\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used, not incl. wired", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "node_memory_wired_bytes{job=\"node\", instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used, wired", + "range": true, + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "node_memory_free_bytes{job=\"node\", instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "range": true, + "refId": "D" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "node_memory_swap_used_bytes{job=\"node\", instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "swap used", + "range": true, + "refId": "E" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(\n node_memory_swap_total_bytes{job=\"node\", instance=\"$node\"}\n-\n node_memory_swap_used_bytes{job=\"node\", instance=\"$node\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "swap free", + "range": true, + "refId": "F" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 32, + "panels": [], + "title": "Settings", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 11 + }, + "id": 40, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_settings_shared_buffers_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Shared Buffers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 11 + }, + "id": 42, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_settings_effective_cache_size_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Effective Cache", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 11 + }, + "id": 44, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_settings_maintenance_work_mem_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Maintenance Work Mem", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 9, + "y": 11 + }, + "id": 46, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_settings_work_mem_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Work Mem", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 12, + "y": 11 + }, + "id": 48, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_settings_max_wal_size_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Max WAL Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 15, + "y": 11 + }, + "id": 50, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_settings_random_page_cost{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Random Page Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 18, + "y": 11 + }, + "id": 52, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_settings_seq_page_cost{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Seq Page Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 20, + "y": 11 + }, + "id": 54, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_settings_max_worker_processes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Max Worker Processes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 22, + "y": 11 + }, + "id": 56, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_settings_max_parallel_workers{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Max Parallel Workers", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 30, + "panels": [], + "title": "Database Stats", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 8, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 1, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_stat_activity_count{datname=~\"$datname\", instance=~\"$instance\", state=\"active\"} !=0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{datname}}, s: {{state}}", + "refId": "A", + "step": 2 + } + ], + "title": "Active sessions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 60, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "irate(pg_stat_database_xact_commit{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}} commits", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "irate(pg_stat_database_xact_rollback{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}} rollbacks", + "refId": "B" + } + ], + "title": "Transactions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 8, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_stat_database_tup_updated{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "title": "Update data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 22 + }, + "id": 5, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_stat_database_tup_fetched{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "title": "Fetch data (SELECT)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 22 + }, + "id": 6, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_stat_database_tup_inserted{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "title": "Insert data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 22 + }, + "id": 3, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_locks_count{datname=~\"$datname\", instance=~\"$instance\", mode=~\"$mode\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}},{{mode}}", + "refId": "A", + "step": 2 + } + ], + "title": "Lock tables", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 31 + }, + "id": 14, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_stat_database_tup_returned{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "title": "Return data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 31 + }, + "id": 71, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_stat_activity_count{datname=~\"$datname\", instance=~\"$instance\", state=~\"idle|idle in transaction|idle in transaction (aborted)\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}, s: {{state}}", + "refId": "A", + "step": 2 + } + ], + "title": "Idle sessions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 31 + }, + "id": 7, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_stat_database_tup_deleted{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "title": "Delete data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 4, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 40 + }, + "id": 62, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "pg_stat_database_blks_hit{instance=\"$instance\", datname=~\"$datname\"} / (pg_stat_database_blks_read{instance=\"$instance\", datname=~\"$datname\"} + pg_stat_database_blks_hit{instance=\"$instance\", datname=~\"$datname\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ datname }}", + "refId": "A" + } + ], + "title": "Cache Hit Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 40 + }, + "id": 64, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "irate(pg_stat_bgwriter_buffers_backend_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_backend", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "irate(pg_stat_bgwriter_buffers_alloc_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_alloc", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "backend_fsync", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "irate(pg_stat_bgwriter_buffers_checkpoint_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_checkpoint", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "irate(pg_stat_bgwriter_buffers_clean_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_clean", + "refId": "E" + } + ], + "title": "Buffers (bgwriter)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 40 + }, + "id": 66, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "irate(pg_stat_database_conflicts{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}} conflicts", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "irate(pg_stat_database_deadlocks{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}} deadlocks", + "refId": "A" + } + ], + "title": "Conflicts/Deadlocks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 16, + "x": 8, + "y": 48 + }, + "id": 70, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "irate(pg_stat_bgwriter_checkpoint_write_time_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "write_time - Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk.", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time_total{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sync_time - Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk.", + "refId": "A" + } + ], + "title": "Checkpoint Stats", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration (minutes)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 1, + "drawStyle": "line", + "fillOpacity": 13, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepBefore", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 5, + "pointSize": 4, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "always", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 20, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "m" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 102, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Name", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "hoverProximity": 0, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "editorMode": "code", + "expr": "increase(cardano_db_sync_db_epoch_sync_number[1m])", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Epoch Sync Duration Timeline - HOW LONG Each Epoch Took", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "description": "Total amount of data written to temporary files by queries in this database. All temporary files are counted, regardless of why the temporary file was created, and regardless of the log_temp_files setting.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 63 + }, + "id": 68, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "beudnd28ss0zkf" + }, + "expr": "irate(pg_stat_database_temp_bytes{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}}", + "refId": "A" + } + ], + "title": "Temp File (Bytes)", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "10s", + "schemaVersion": 41, + "tags": [ + "postgres", + "db", + "stats" + ], + "templating": { + "list": [ + { + "current": { + "text": "", + "value": "" + }, + "datasource": "beudnd28ss0zkf", + "definition": "", + "includeAll": false, + "label": "Release", + "name": "release", + "options": [], + "query": "query_result(pg_exporter_last_scrape_duration_seconds{kubernetes_namespace=\"$namespace\"})", + "refresh": 2, + "regex": "/.*release=\"([^\"]+)/", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "localhost:9187", + "value": "localhost:9187" + }, + "datasource": "beudnd28ss0zkf", + "definition": "query_result(pg_up{release=\"$release\"})", + "includeAll": false, + "label": "Instance", + "name": "instance", + "options": [], + "query": { + "qryType": 3, + "query": "query_result(pg_up{release=\"$release\"})", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/.*instance=\"([^\"]+).*/", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "beudnd28ss0zkf", + "definition": "", + "includeAll": true, + "label": "Database", + "multi": true, + "name": "datname", + "options": [], + "query": "label_values(datname)", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "beudnd28ss0zkf", + "definition": "", + "includeAll": true, + "label": "Lock table", + "multi": true, + "name": "mode", + "options": [], + "query": "label_values({mode=~\"accessexclusivelock|accesssharelock|exclusivelock|rowexclusivelock|rowsharelock|sharelock|sharerowexclusivelock|shareupdateexclusivelock\"}, mode)", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "", + "value": "" + }, + "definition": "label_values(node_exporter_build_info{job=\"node\"},instance)", + "label": "node", + "name": "node", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(node_exporter_build_info{job=\"node\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Cardano-db-sync", + "uid": "000000039", + "version": 36 +} diff --git a/dev-tools/monitoring/config/prometheus.yml b/dev-tools/monitoring/config/prometheus.yml new file mode 100644 index 000000000..29c7604a8 --- /dev/null +++ b/dev-tools/monitoring/config/prometheus.yml @@ -0,0 +1,42 @@ +# Prometheus Configuration for Cardano DB Sync Monitoring + +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + monitor: 'cardano-db-sync-dev' + +# Scrape configurations +scrape_configs: + # PostgreSQL metrics via postgres_exporter + - job_name: 'postgres-cexplorer' + static_configs: + - targets: ['localhost:9187'] + labels: + instance: 'cardano-db-sync-db' + environment: 'development' + scrape_interval: 5s + + # System metrics via node_exporter + - job_name: 'node' + static_configs: + - targets: ['localhost:9100'] + labels: + instance: 'cardano-db-sync-host' + environment: 'development' + scrape_interval: 10s + + # Cardano DB Sync application metrics + - job_name: 'cardano-db-sync' + static_configs: + - targets: ['localhost:8080'] + labels: + instance: 'cardano-db-sync' + environment: 'development' + scrape_interval: 10s + # This endpoint will expose: + # - Sync progress metrics + # - Database operation metrics + # - Memory/GC metrics from ghc-debug integration + # - Cache hit/miss rates + # - Transaction processing rates diff --git a/dev-tools/monitoring/docs/METRICS.md b/dev-tools/monitoring/docs/METRICS.md new file mode 100644 index 000000000..7449bb907 --- /dev/null +++ b/dev-tools/monitoring/docs/METRICS.md @@ -0,0 +1,126 @@ +# Cardano DB Sync - Available Metrics + +This document describes the metrics exposed by the monitoring suite for cardano-db-sync. + +## Metric Sources + +### 1. PostgreSQL Metrics (postgres_exporter) + +Exposed on port **9187** via `postgres_exporter`. + +#### Database Size Metrics +- `pg_database_size_bytes{datname="cexplorer"}` - Total database size in bytes + +#### Connection Metrics +- `pg_stat_database_numbackends` - Number of active connections +- `pg_stat_database_xact_commit` - Total transactions committed +- `pg_stat_database_xact_rollback` - Total transactions rolled back +- `pg_stat_database_deadlocks` - Number of deadlocks detected + +#### Table Metrics +- `pg_stat_user_tables_seq_scan` - Sequential scans on tables +- `pg_stat_user_tables_idx_scan` - Index scans on tables +- `pg_stat_user_tables_n_tup_ins` - Rows inserted +- `pg_stat_user_tables_n_tup_upd` - Rows updated +- `pg_stat_user_tables_n_tup_del` - Rows deleted +- `pg_stat_user_tables_n_live_tup` - Estimated live rows +- `pg_stat_user_tables_n_dead_tup` - Estimated dead rows + +#### Index Metrics +- `pg_stat_user_indexes_idx_scan` - Index scans performed +- `pg_stat_user_indexes_idx_tup_read` - Index entries returned +- `pg_stat_user_indexes_idx_tup_fetch` - Live rows fetched by index scans + +#### Transaction/WAL Metrics +- `pg_stat_database_blks_read` - Disk blocks read +- `pg_stat_database_blks_hit` - Disk blocks found in cache (buffer hit) +- `pg_stat_database_tup_returned` - Rows returned by queries +- `pg_stat_database_tup_fetched` - Rows fetched by queries +- `pg_stat_database_tup_inserted` - Rows inserted +- `pg_stat_database_tup_updated` - Rows updated +- `pg_stat_database_tup_deleted` - Rows deleted + +#### Replication Metrics (if applicable) +- `pg_stat_replication_lag` - Replication lag in bytes + +### 2. System Metrics (node_exporter) + +Exposed on port **9100** via `node_exporter`. + +#### CPU Metrics +- `node_cpu_seconds_total` - CPU time spent in various modes (user, system, idle, etc.) +- `node_load1`, `node_load5`, `node_load15` - System load averages + +#### Memory Metrics +- `node_memory_MemTotal_bytes` - Total memory +- `node_memory_MemFree_bytes` - Free memory +- `node_memory_MemAvailable_bytes` - Available memory +- `node_memory_Buffers_bytes` - Memory used for buffers +- `node_memory_Cached_bytes` - Memory used for cache +- `node_memory_SwapTotal_bytes` - Total swap space +- `node_memory_SwapFree_bytes` - Free swap space + +#### Disk Metrics +- `node_disk_read_bytes_total` - Total bytes read from disk +- `node_disk_written_bytes_total` - Total bytes written to disk +- `node_disk_read_time_seconds_total` - Time spent reading +- `node_disk_write_time_seconds_total` - Time spent writing +- `node_filesystem_size_bytes` - Filesystem size +- `node_filesystem_avail_bytes` - Filesystem space available +- `node_filesystem_free_bytes` - Filesystem space free + +#### Network Metrics +- `node_network_receive_bytes_total` - Network bytes received +- `node_network_transmit_bytes_total` - Network bytes transmitted +- `node_network_receive_errs_total` - Network receive errors +- `node_network_transmit_errs_total` - Network transmit errors + +### 3. Cardano DB Sync Application Metrics + +Exposed on port **8080** via Prometheus endpoint in cardano-db-sync. + +**Status**: To be implemented when Prometheus endpoint is added to cardano-db-sync. + +#### Planned Sync Metrics +- `dbsync_block_height` - Current synced block height +- `dbsync_slot_number` - Current synced slot number +- `dbsync_epoch_number` - Current synced epoch number +- `dbsync_sync_progress_percent` - Sync progress percentage +- `dbsync_blocks_per_second` - Block processing rate +- `dbsync_tx_per_second` - Transaction processing rate +- `dbsync_rollback_count` - Number of rollbacks performed +- `dbsync_rollback_depth` - Depth of most recent rollback + +#### Planned Database Operation Metrics +- `dbsync_db_insert_duration_seconds` - Histogram of insert operation durations +- `dbsync_db_query_duration_seconds` - Histogram of query operation durations +- `dbsync_db_bulk_insert_size` - Size of bulk insert batches +- `dbsync_grouped_data_flush_duration_seconds` - Time spent flushing grouped data +- `dbsync_grouped_data_size_bytes` - Size of grouped data in memory + +#### Planned Cache Metrics +- `dbsync_cache_hits_total` - Cache hits by cache type +- `dbsync_cache_misses_total` - Cache misses by cache type +- `dbsync_cache_size_entries` - Current cache size in entries +- `dbsync_cache_size_bytes` - Estimated cache size in bytes +- `dbsync_cache_evictions_total` - Number of cache evictions + +#### Planned Memory/GC Metrics (from ghc-debug integration) +- `dbsync_memory_heap_size_bytes` - Current heap size +- `dbsync_memory_live_bytes` - Live data in heap +- `dbsync_memory_gc_count` - Number of GC collections +- `dbsync_memory_gc_cpu_seconds` - CPU time spent in GC +- `dbsync_memory_gc_wall_seconds` - Wall time spent in GC +- `dbsync_memory_max_live_bytes` - Maximum live data observed +- `dbsync_memory_allocated_bytes_total` - Total bytes allocated + +#### Planned Ledger State Metrics +- `dbsync_ledger_state_size_bytes` - Size of ledger state +- `dbsync_ledger_snapshot_duration_seconds` - Time to save ledger snapshot +- `dbsync_ledger_events_processed_total` - Ledger events processed + +## See Also + +- [Prometheus Query Documentation](https://prometheus.io/docs/prometheus/latest/querying/basics/) +- [PostgreSQL Statistics Views](https://www.postgresql.org/docs/current/monitoring-stats.html) +- [Node Exporter Metrics](https://github.com/prometheus/node_exporter#enabled-by-default) diff --git a/dev-tools/monitoring/scripts/start-monitoring.sh b/dev-tools/monitoring/scripts/start-monitoring.sh new file mode 100755 index 000000000..0adbc5cdb --- /dev/null +++ b/dev-tools/monitoring/scripts/start-monitoring.sh @@ -0,0 +1,197 @@ +#!/bin/bash + +set -euo pipefail + +# Cardano DB Sync - Monitoring Suite Launcher +# Starts Prometheus, postgres_exporter, and node_exporter in a tmux session + +SESSION="cardano-monitoring" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MONITORING_DIR="$(dirname "$SCRIPT_DIR")" +CONFIG_DIR="$MONITORING_DIR/config" +DATA_DIR="$MONITORING_DIR/data" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +print_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +print_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check dependencies +check_dependencies() { + local missing=() + + command -v tmux >/dev/null 2>&1 || missing+=("tmux") + command -v prometheus >/dev/null 2>&1 || missing+=("prometheus") + command -v postgres_exporter >/dev/null 2>&1 || missing+=("postgres_exporter") + command -v node_exporter >/dev/null 2>&1 || missing+=("node_exporter") + + if [ ${#missing[@]} -gt 0 ]; then + print_error "Missing required dependencies: ${missing[*]}" + print_info "Install with:" + echo " brew install tmux prometheus postgres_exporter node_exporter" + exit 1 + fi +} + +# Check if PostgreSQL is running +check_postgres() { + if ! pg_isready -h 127.0.0.1 -p 5432 >/dev/null 2>&1; then + print_warn "PostgreSQL does not appear to be running on localhost:5432" + print_info "Start PostgreSQL before running monitoring suite" + read -p "Continue anyway? [y/N] " -n 1 -r + echo + [[ ! $REPLY =~ ^[Yy]$ ]] && exit 1 + fi +} + +# Create data directory +ensure_data_dir() { + mkdir -p "$DATA_DIR" + print_info "Data directory: $DATA_DIR" +} + +# Kill existing session if it exists +cleanup_existing() { + if tmux has-session -t "$SESSION" 2>/dev/null; then + print_warn "Killing existing session: $SESSION" + tmux kill-session -t "$SESSION" + + # Kill any orphaned exporters/prometheus + pkill -f postgres_exporter 2>/dev/null || true + pkill -f node_exporter 2>/dev/null || true + pkill -f prometheus 2>/dev/null || true + + sleep 1 + fi +} + +# Database connection settings +DB_HOST="${DB_HOST:-127.0.0.1}" +DB_PORT="${DB_PORT:-5432}" +DB_NAME="${DB_NAME:-cexplorer}" +DB_USER="${DB_USER:-postgres_exporter}" + +# Exporter ports +POSTGRES_EXPORTER_PORT="${POSTGRES_EXPORTER_PORT:-9187}" +NODE_EXPORTER_PORT="${NODE_EXPORTER_PORT:-9100}" +PROMETHEUS_PORT="${PROMETHEUS_PORT:-9090}" + +print_info "Starting Cardano DB Sync Monitoring Suite" +print_info "==========================================" + +check_dependencies +check_postgres +ensure_data_dir +cleanup_existing + +# Create new tmux session +tmux new-session -d -s "$SESSION" + +# Configure layout: 4 panes +# β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +# β”‚ 0 β”‚ 2 β”‚ +# β”‚ postgresβ”‚ node β”‚ +# β”‚ exporterβ”‚ exporterβ”‚ +# β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +# β”‚ 1 β”‚ 3 β”‚ +# β”‚prometheusβ”‚ info β”‚ +# β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +tmux rename-window -t "$SESSION" "monitoring" +tmux split-window -h -t "$SESSION" +tmux split-window -v -t "$SESSION:0.0" +tmux split-window -v -t "$SESSION:0.2" + +# Pane 0: postgres_exporter +print_info "Starting postgres_exporter on port $POSTGRES_EXPORTER_PORT" +tmux send-keys -t "$SESSION:0.0" "cd '$MONITORING_DIR'" C-m +tmux send-keys -t "$SESSION:0.0" "echo '=== PostgreSQL Exporter ==='" C-m +tmux send-keys -t "$SESSION:0.0" "export DATA_SOURCE_NAME='postgresql://${DB_USER}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable'" C-m +tmux send-keys -t "$SESSION:0.0" "postgres_exporter --web.listen-address=:${POSTGRES_EXPORTER_PORT}" C-m + +# Pane 1: prometheus +print_info "Starting Prometheus on port $PROMETHEUS_PORT" +tmux send-keys -t "$SESSION:0.1" "cd '$MONITORING_DIR'" C-m +tmux send-keys -t "$SESSION:0.1" "echo '=== Prometheus ==='" C-m +tmux send-keys -t "$SESSION:0.1" "prometheus --config.file='$CONFIG_DIR/prometheus.yml' --storage.tsdb.path='$DATA_DIR' --web.listen-address=:${PROMETHEUS_PORT}" C-m + +# Pane 2: node_exporter +print_info "Starting node_exporter on port $NODE_EXPORTER_PORT" +tmux send-keys -t "$SESSION:0.2" "cd '$MONITORING_DIR'" C-m +tmux send-keys -t "$SESSION:0.2" "echo '=== Node Exporter ==='" C-m +tmux send-keys -t "$SESSION:0.2" "node_exporter --web.listen-address=:${NODE_EXPORTER_PORT} --no-collector.thermal" C-m + +# Pane 3: Info pane +tmux send-keys -t "$SESSION:0.3" "cd '$MONITORING_DIR'" C-m +tmux send-keys -t "$SESSION:0.3" "clear" C-m +tmux send-keys -t "$SESSION:0.3" "cat << 'EOF' +╔═══════════════════════════════════════════════════════════════╗ +β•‘ Cardano DB Sync Monitoring Suite β•‘ +β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• + +Monitoring Status: + βœ“ PostgreSQL Exporter : http://localhost:${POSTGRES_EXPORTER_PORT} + βœ“ Node Exporter : http://localhost:${NODE_EXPORTER_PORT} + βœ“ Prometheus : http://localhost:${PROMETHEUS_PORT} + +Quick Links: + - Prometheus UI : http://localhost:${PROMETHEUS_PORT} + - Metrics Explorer : http://localhost:${PROMETHEUS_PORT}/graph + - Targets Status : http://localhost:${PROMETHEUS_PORT}/targets + +Grafana Setup: + 1. Install Grafana: brew install grafana + 2. Start Grafana: brew services start grafana + 3. Access: http://localhost:3000 (admin/admin) + 4. Add Prometheus datasource: http://localhost:${PROMETHEUS_PORT} + 5. Import dashboard from docs/grafana-dashboard.json + +Useful Commands: + - View Prometheus config : cat config/prometheus.yml + - View logs : Check panes above + - Stop monitoring : tmux kill-session -t ${SESSION} + - Detach from session : Ctrl+b, d + - Re-attach to session : tmux attach -t ${SESSION} + +Database Connection: + Host : ${DB_HOST} + Port : ${DB_PORT} + Database : ${DB_NAME} + User : ${DB_USER} + +Documentation: + - See docs/README.md for full setup guide + - See docs/METRICS.md for available metrics + - See docs/grafana-dashboard.json for Grafana template + +EOF +" C-m + +sleep 2 + +print_info "" +print_info "Monitoring suite started successfully!" +print_info "" +print_info " Prometheus UI : http://localhost:${PROMETHEUS_PORT}" +print_info " Targets : http://localhost:${PROMETHEUS_PORT}/targets" +print_info "" +print_info "Attaching to tmux session '$SESSION'..." +print_info "(Press Ctrl+b, d to detach; tmux attach -t $SESSION to reattach)" +print_info "" + +# Attach to session +tmux select-pane -t "$SESSION:0.3" +tmux attach-session -t "$SESSION" diff --git a/dev-tools/profiling/README.md b/dev-tools/profiling/README.md new file mode 100644 index 000000000..c8b04447a --- /dev/null +++ b/dev-tools/profiling/README.md @@ -0,0 +1,94 @@ +# Cardano DB Sync - ghc-debug Profiling + +Interactive heap profiling for cardano-db-sync using ghc-debug. + +## Quick Setup + +### 1. Install ghc-debug-brick + +```bash +cabal install ghc-debug-brick +``` + +This automatically installs all ghc-debug dependencies. + +### 2. Configure ghc-debug (first time only) + +The ghc-debug dependencies are configured in `cabal.project.local` (not committed to git). + +If you don't have them yet, add to your `cabal.project.local`: + +```cabal +source-repository-package + type: git + location: https://gitlab.haskell.org/ghc/ghc-debug.git + tag: 1b0f36fab86e9baa9734c88dcc1dbe17d10d8c93 + subdir: stub + +source-repository-package + type: git + location: https://gitlab.haskell.org/ghc/ghc-debug.git + tag: 1b0f36fab86e9baa9734c88dcc1dbe17d10d8c93 + subdir: common +``` + +**Note**: The ghc-debug commit above is from master branch (0.7.0.0). If you have GHC compatibility issues, you may need to adjust this commit. + +### 3. Build cardano-db-sync with profiling + +```bash +# From repository root +cabal build --flag enable-ghc-debug exe:cardano-db-sync + +# Or use the build script +./dev-tools/profiling/scripts/build-profiling.sh +``` + +### 4. Run cardano-db-sync + +Start cardano-db-sync as you normally would. The `enable-ghc-debug` flag automatically enables ghc-debug support: + +```bash +# Using the start script (launches in tmux) +./dev-tools/profiling/scripts/start-profiling.sh + +# Or run manually with your usual command +PGPASSFILE=config/pgpass-mainnet cabal run cardano-db-sync -- \ + --config config/mainnet-config.yaml \ + --socket-path /path/to/node.socket \ + --state-dir ledger-state/mainnet \ + --schema-dir schema/ +``` + +**Note**: When running with `enable-ghc-debug`, cardano-db-sync will print: +``` +Starting ghc-debug on socket: ~/.local/share/ghc-debug/debuggee/sockets/-cardano-db-sync +``` + +### 5. Connect with ghc-debug-brick + +In a new terminal, while cardano-db-sync is running: + +```bash +ghc-debug-brick +``` + +## Files + +- `scripts/build-profiling.sh` - Build with profiling enabled +- `scripts/start-profiling.sh` - Launch cardano-db-sync in tmux with monitoring + +## Troubleshooting + +**ghc-debug-brick shows "0 found"**: +1. Run `ghc-debug-brick` without arguments +2. Press `Tab` to switch to process view +3. Check socket exists: `ls ~/.local/share/ghc-debug/debuggee/sockets/` +4. Verify cardano-db-sync is running: `ps aux | grep cardano-db-sync` + +**MacOS Do NOT set `GHC_DEBUG_SOCKET` environment variable** - it can break auto-discovery. + +## See Also + +- [ghc-debug Documentation](https://gitlab.haskell.org/ghc/ghc-debug) +- [ouroboros-consensus PR #1731](https://github.com/IntersectMBO/ouroboros-consensus/pull/1731) - Similar approach diff --git a/dev-tools/profiling/scripts/build-profiling.sh b/dev-tools/profiling/scripts/build-profiling.sh new file mode 100755 index 000000000..991793a04 --- /dev/null +++ b/dev-tools/profiling/scripts/build-profiling.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# build-profiling.sh +# Build cardano-db-sync with ghc-debug profiling support enabled + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROFILING_DIR="$(dirname "$SCRIPT_DIR")" +DEV_TOOLS_DIR="$(dirname "$PROFILING_DIR")" +CARDANO_DB_SYNC_DIR="$(dirname "$DEV_TOOLS_DIR")" + +cd "$CARDANO_DB_SYNC_DIR" + +echo "Building cardano-db-sync with profiling support..." +echo "" + +cabal build --flag enable-ghc-debug exe:cardano-db-sync + +DBSYNC_BIN=$(cabal list-bin exe:cardano-db-sync --flag enable-ghc-debug) + +echo "" +echo "Build complete: $DBSYNC_BIN" +echo "" diff --git a/dev-tools/profiling/scripts/start-profiling.sh b/dev-tools/profiling/scripts/start-profiling.sh new file mode 100755 index 000000000..63d718706 --- /dev/null +++ b/dev-tools/profiling/scripts/start-profiling.sh @@ -0,0 +1,123 @@ +#!/bin/bash +# start-profiling.sh +# Run cardano-db-sync with ghc-debug profiling enabled +# Based on ouroboros-consensus team's approach + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROFILING_DIR="$(dirname "$SCRIPT_DIR")" +DEV_TOOLS_DIR="$(dirname "$PROFILING_DIR")" +CARDANO_DB_SYNC_DIR="$(dirname "$DEV_TOOLS_DIR")" + +HOMEIOG=${HOMEIOG:-$HOME/Code/IOG} +CARDANO_NODE_DIR="${CARDANO_NODE_DIR:-$HOMEIOG/cardano-node}" +TESTNET_DIR="${TESTNET_DIR:-$HOMEIOG/testnet}" + +# Detect OS for pgpass file +if [[ "$OSTYPE" == "darwin"* ]]; then + PGPASS_FILE="${PGPASS_FILE:-$CARDANO_DB_SYNC_DIR/config/pgpass-mainnet-macos}" +else + PGPASS_FILE="${PGPASS_FILE:-$CARDANO_DB_SYNC_DIR/config/pgpass-mainnet}" +fi + +# Profiling configuration +SNAPSHOT_DIR="$PROFILING_DIR/snapshots" +LOG_DIR="$PROFILING_DIR/logs" +MEMORY_LOG="$LOG_DIR/memory-$(date +%Y%m%d-%H%M%S).csv" +# Note: Don't set GHC_DEBUG_SOCKET - let ghc-debug-stub use default XDG location +# Socket will be at: ~/.local/share/ghc-debug/debuggee/sockets/-cardano-db-sync + +# Create profiling directories +mkdir -p "$SNAPSHOT_DIR" +mkdir -p "$LOG_DIR" + +# Build cardano-db-sync with profiling support +echo "Building profiling-enabled binary..." +"$SCRIPT_DIR/build-profiling.sh" +echo "" + +# Find the profiling-enabled binary (similar to run-everything-tmux.sh) +dbsync="$(find "$CARDANO_DB_SYNC_DIR"/ -name cardano-db-sync -type f | head -1)" + +if [ -z "$dbsync" ]; then + echo "ERROR: Could not find cardano-db-sync binary" + echo "Build may have failed. Check output above." + exit 1 +fi + +echo "======================================" +echo "cardano-db-sync Profiling Setup" +echo "======================================" +echo "DB Sync binary: $dbsync" +echo "Memory log: $MEMORY_LOG" +echo "Snapshot dir: $SNAPSHOT_DIR" +echo "Socket location: ~/.local/share/ghc-debug/debuggee/sockets/-cardano-db-sync" +echo "======================================" +echo "" + +# Check if ghc-debug-brick is installed +if ! command -v ghc-debug-brick &> /dev/null; then + echo "WARNING: ghc-debug-brick not found in PATH" + echo "To install:" + echo " git clone https://gitlab.haskell.org/ghc/ghc-debug.git" + echo " cd ghc-debug/brick" + echo " cabal install ghc-debug-brick" + echo "" +fi + +# Note: Sockets are auto-managed by ghc-debug in XDG directory + +# Set up tmux session +session="DB-SYNC-PROFILE" + +# Kill existing session if it exists +if tmux has-session -t $session 2>/dev/null; then + echo "Killing existing tmux session: $session" + tmux kill-session -t $session + killall cardano-node 2>/dev/null || true + pkill -f cardano-db-sync 2>/dev/null || true + sleep 2 +fi + +echo "Creating tmux session: $session" +tmux new-session -d -s $session + +# Rename the window +tmux rename-window -t $session "profiling" + +# Split into 3 panes: +# 0: cardano-node +# 1: cardano-db-sync +# 2: monitoring +tmux split-window -h -t $session +tmux split-window -v -t $session:0.1 + +# Pane 0: Cardano Node +echo "Setting up cardano-node (pane 0)..." +tmux send-keys -t $session:0.0 "cd $CARDANO_NODE_DIR/" 'C-m' +tmux send-keys -t $session:0.0 "echo 'Starting cardano-node...'" 'C-m' +tmux send-keys -t $session:0.0 "cardano-node run --config $TESTNET_DIR/config.json --database-path $TESTNET_DIR/db/ --socket-path $TESTNET_DIR/db/node.socket --host-addr 0.0.0.0 --port 1337 --topology $TESTNET_DIR/topology.json" 'C-m' + +# Wait for node to start +sleep 5 + +# Pane 1: Cardano DB-Sync with profiling +echo "Setting up cardano-db-sync with profiling (pane 1)..." +tmux send-keys -t $session:0.1 "cd $CARDANO_DB_SYNC_DIR/" 'C-m' +tmux send-keys -t $session:0.1 "export PGPASSFILE=$PGPASS_FILE" 'C-m' +sleep 2 +tmux send-keys -t $session:0.1 "PGPASSFILE=$PGPASS_FILE $dbsync --config $TESTNET_DIR/db-sync-config.json --socket-path $TESTNET_DIR/db/node.socket --state-dir $TESTNET_DIR/ledger-state --schema-dir $CARDANO_DB_SYNC_DIR/schema/" 'C-m' + +# Pane 2: Monitoring and instructions +echo "Setting up monitoring pane (pane 2)..." +tmux send-keys -t $session:0.2 "cd $CARDANO_DB_SYNC_DIR/" 'C-m' + +# Start memory monitoring +tmux send-keys -t $session:0.2 "$SCRIPT_DIR/monitor-dbsync-memory.sh $MEMORY_LOG 300 2>&1 | tee -a $LOG_DIR/monitor.log" 'C-m' + +# Set pane sizes +tmux resize-pane -t $session:0.0 -x 50% +tmux resize-pane -t $session:0.1 -x 50% + +tmux -CC attach-session -t $session diff --git a/scripts/run-everything-tmux.sh b/scripts/run-everything-tmux.sh index c6d845c24..1c78ba1c1 100755 --- a/scripts/run-everything-tmux.sh +++ b/scripts/run-everything-tmux.sh @@ -3,7 +3,7 @@ HOMEIOG=$HOME/Code/IOG current_dir=$(basename "$PWD") CARDANO_NODE_DIR="$HOMEIOG/cardano-node" -CARDANO_DB_SYNC_DIR="$HOMEIOG/$current_dir" +CARDANO_DB_SYNC_DIR="ng $HOMEIOG/$current_dir" TESTNET_DIR="$HOMEIOG/testnet" dbsync="$(find "$CARDANO_DB_SYNC_DIR"/ -name cardano-db-sync -type f)" @@ -29,7 +29,6 @@ tmux send-keys -t 0 "cardano-node run --config $TESTNET_DIR/config.json --databa tmux send-keys -t 1 "cd $CARDANO_DB_SYNC_DIR/" 'C-m'; sleep 3 tmux send-keys -t 1 "export PGPASSFILE=$CARDANO_DB_SYNC_DIR/config/pgpass-mainnet-macos" 'C-m'; sleep 2 tmux send-keys -t 1 "PGPASSFILE=$CARDANO_DB_SYNC_DIR/config/pgpass-mainnet-macos $dbsync --config $TESTNET_DIR/db-sync-config.json --socket-path $TESTNET_DIR/db/node.socket --state-dir $TESTNET_DIR/ledger-state --schema-dir $CARDANO_DB_SYNC_DIR/schema/" 'C-m' -# tmux send-keys -t 1 "$dbsync --config $TESTNET_DIR/db-sync-config.json --socket-path $TESTNET_DIR/db/node.socket --state-dir $TESTNET_DIR/ledger-state --schema-dir $CARDANO_DB_SYNC_DIR/schema/ +RTS -p -hc -L200 -RTS" 'C-m' +# tmux send-keys -t 1 "PGPASSFILE=$CARDANO_DB_SYNC_DIR/config/pgpass-mainnet-macos $dbsync --config $TESTNET_DIR/db-sync-config.json --socket-path $TESTNET_DIR/db/node.socket --state-dir $TESTNET_DIR/ledger-state --schema-dir $CARDANO_DB_SYNC_DIR/schema/ +RTS -l -p -hc -L200 -RTS" 'C-m' tmux -CC attach-session -t $session -# tmux send-keys -t 1 "$dbsync --config $HOMEIOG/testnet/db-sync-config.json --socket-path $HOMEIOG/testnet/db/node.socket --state-dir $HOMEIOG/testnet/ledger-state --schema-dir $HOMEIOG/cardano-db-sync/schema/ +RTS -p -hc -L200 -RTS" 'C-m'