Skip to content

Commit 620fa32

Browse files
cali-jumptradingcali-jumptrading
committed
snapshots: lthash tile
1 parent 1fb3336 commit 620fa32

28 files changed

+967
-99
lines changed

‎book/api/metrics-generated.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -885,6 +885,18 @@
885885

886886
</div>
887887

888+
## Snaplt Tile
889+
890+
<div class="metrics">
891+
892+
| Metric | Type | Description |
893+
|--------|------|-------------|
894+
| <span class="metrics-name">snaplt_&#8203;state</span> | gauge | State of the tile. 0=hashing, 1=done, 2=shutdown |
895+
| <span class="metrics-name">snaplt_&#8203;full_&#8203;accounts_&#8203;hashed</span> | gauge | Number of accounts hashed for the full snapshot during snapshot loading. Might decrease if snapshot load is aborted and restarted |
896+
| <span class="metrics-name">snaplt_&#8203;incremental_&#8203;accounts_&#8203;hashed</span> | gauge | Number of accounts hashed for the incremental snapshot during snapshot loading. Might decrease if snapshot load is aborted and restarted |
897+
898+
</div>
899+
888900
## Ipecho Tile
889901

890902
<div class="metrics">

‎src/app/firedancer-dev/commands/backtest.c‎

Lines changed: 77 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "../../../disco/metrics/fd_metrics.h"
2424
#include "../../../util/pod/fd_pod_format.h"
2525
#include "../../../discof/restore/utils/fd_ssmsg.h"
26+
#include "../../../discof/restore/utils/fd_ssctrl.h"
2627
#include "../../../discof/tower/fd_tower_tile.h"
2728
#include "../../../discof/reasm/fd_reasm.h"
2829
#include "../../../discof/replay/fd_exec.h" /* FD_RUNTIME_PUBLIC_ACCOUNT_UPDATE_MSG_MTU */
@@ -38,6 +39,8 @@ static void
3839
backtest_topo( config_t * config ) {
3940
ulong exec_tile_cnt = config->firedancer.layout.exec_tile_count;
4041
ulong writer_tile_cnt = config->firedancer.layout.writer_tile_count;
42+
ulong snaplt_tile_cnt = config->firedancer.layout.snaplt_tile_count;
43+
int snaplt_disabled = config->development.snapshots.disable_lthash_verification;
4144

4245
int disable_snap_loader = !config->gossip.entrypoints_cnt;
4346
int solcap_enabled = strlen( config->capture.solcap_capture )>0;
@@ -100,6 +103,14 @@ backtest_topo( config_t * config ) {
100103
snaprd_tile->allow_shutdown = 1;
101104
snapdc_tile->allow_shutdown = 1;
102105
snapin_tile->allow_shutdown = 1;
106+
107+
if( FD_LIKELY( !snaplt_disabled ) ) {
108+
fd_topob_wksp( topo, "snaplt" );
109+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
110+
fd_topo_tile_t * snaplt_tile = fd_topob_tile( topo, "snaplt", "snaplt", "metric_in", cpu_idx++, 0, 0 );
111+
snaplt_tile->allow_shutdown = 1;
112+
}
113+
}
103114
} else {
104115
fd_topob_wksp( topo, "genesi" );
105116
fd_topob_tile( topo, "genesi", "genesi", "metric_in", cpu_idx++, 0, 0 )->allow_shutdown = 1;
@@ -128,7 +139,11 @@ backtest_topo( config_t * config ) {
128139
fd_topob_wksp( topo, "snapdc_rd" );
129140
fd_topob_wksp( topo, "snapin_rd" );
130141
fd_topob_wksp( topo, "snap_out" );
131-
fd_topob_wksp( topo, "replay_manif" );
142+
143+
if( FD_LIKELY( !snaplt_disabled ) ) {
144+
fd_topob_wksp( topo, "snapin_lt" );
145+
fd_topob_wksp( topo, "snaplt_out" );
146+
}
132147
/* TODO: Should be depth of 1 or 2, not 4, but it causes backpressure
133148
from the replay tile parsing the manifest, remove when this is
134149
fixed. */
@@ -139,6 +154,11 @@ backtest_topo( config_t * config ) {
139154
fd_topob_link( topo, "snapdc_rd", "snapdc_rd", 128UL, 0UL, 1UL );
140155
fd_topob_link( topo, "snapin_rd", "snapin_rd", 128UL, 0UL, 1UL );
141156

157+
if( FD_LIKELY( !snaplt_disabled ) ) {
158+
fd_topob_link( topo, "snapin_lt", "snapin_lt", 128UL, sizeof(fd_snapshot_existing_account_t), 1UL );
159+
FOR(snaplt_tile_cnt) fd_topob_link( topo, "snaplt_out", "snaplt_out", 128UL, 2048UL, 1UL );
160+
}
161+
142162
fd_topob_tile_out( topo, "snaprd", 0UL, "snap_zstd", 0UL );
143163
fd_topob_tile_in ( topo, "snapdc", 0UL, "metric_in", "snap_zstd", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
144164
fd_topob_tile_out( topo, "snapdc", 0UL, "snap_stream", 0UL );
@@ -150,6 +170,13 @@ backtest_topo( config_t * config ) {
150170
fd_topob_tile_out( topo, "snapdc", 0UL, "snapdc_rd", 0UL );
151171
fd_topob_tile_in( topo, "snaprd", 0UL, "metric_in", "snapin_rd", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
152172
fd_topob_tile_out( topo, "snapin", 0UL, "snapin_rd", 0UL );
173+
174+
if( FD_LIKELY( !snaplt_disabled ) ) {
175+
fd_topob_tile_out( topo, "snapin", 0UL, "snapin_lt", 0UL );
176+
FOR(snaplt_tile_cnt) fd_topob_tile_in ( topo, "snapin", 0UL, "metric_in", "snaplt_out", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
177+
FOR(snaplt_tile_cnt) fd_topob_tile_in ( topo, "snaplt", i, "metric_in", "snapin_lt", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
178+
FOR(snaplt_tile_cnt) fd_topob_tile_out( topo, "snaplt", i, "snaplt_out", i );
179+
}
153180
} else {
154181
fd_topob_wksp( topo, "genesi_out" );
155182
fd_topob_link( topo, "genesi_out", "genesi_out", 2UL, 10UL*1024UL*1024UL+32UL+sizeof(fd_lthash_value_t), 1UL );
@@ -302,14 +329,7 @@ backtest_topo( config_t * config ) {
302329
}
303330

304331
if( FD_LIKELY( !disable_snap_loader ) ) {
305-
/* Replay decoded manifest dcache topo obj */
306-
fd_topo_obj_t * replay_manifest_dcache = fd_topob_obj( topo, "dcache", "replay_manif" );
307-
fd_pod_insertf_ulong( topo->props, 2UL << 30UL, "obj.%lu.data_sz", replay_manifest_dcache->id );
308-
fd_pod_insert_ulong( topo->props, "manifest_dcache", replay_manifest_dcache->id );
309-
310332
fd_topob_tile_uses( topo, snapin_tile, funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
311-
fd_topob_tile_uses( topo, snapin_tile, replay_manifest_dcache, FD_SHMEM_JOIN_MODE_READ_WRITE );
312-
fd_topob_tile_uses( topo, replay_tile, replay_manifest_dcache, FD_SHMEM_JOIN_MODE_READ_ONLY );
313333
}
314334

315335
for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
@@ -394,23 +414,51 @@ backtest_cmd_fn( args_t * args FD_PARAM_UNUSED,
394414
ulong volatile * const snapdc_metrics = fd_metrics_tile( snapdc_tile->metrics );
395415
ulong volatile * const snapin_metrics = fd_metrics_tile( snapin_tile->metrics );
396416

417+
ulong volatile * snaplt_metrics[ FD_MAX_SNAPLT_TILES ];
418+
ulong snaplt_tile_cnt = fd_topo_tile_name_cnt( topo, "snaplt" );
419+
420+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
421+
ulong snaplt_tile_idx = fd_topo_find_tile( topo, "snaplt", i );
422+
FD_TEST( snaplt_tile_idx!=ULONG_MAX );
423+
fd_topo_tile_t * snaplt_tile = &topo->tiles[ snaplt_tile_idx ];
424+
snaplt_metrics[ i ] = fd_metrics_tile( snaplt_tile->metrics );
425+
}
426+
397427
ulong total_off_old = 0UL;
398428
ulong snaprd_backp_old = 0UL;
399429
ulong snaprd_wait_old = 0UL;
400430
ulong snapdc_backp_old = 0UL;
401431
ulong snapdc_wait_old = 0UL;
402432
ulong snapin_backp_old = 0UL;
403433
ulong snapin_wait_old = 0UL;
434+
ulong snaplt_backp_old = 0UL;
435+
ulong snaplt_wait_old = 0UL;
404436
ulong acc_cnt_old = 0UL;
405437
sleep( 1 );
406-
puts( "-------------backp=(snaprd,snapdc,snapin) busy=(snaprd,snapdc,snapin)---------------" );
438+
puts( "" );
439+
puts( "Columns:" );
440+
puts( "- bw: Uncompressed bandwidth" );
441+
puts( "- backp: Backpressured by downstream tile" );
442+
puts( "- stall: Waiting on upstream tile" );
443+
puts( "- acc: Number of accounts" );
444+
puts( "" );
445+
puts( "-------------backp=(snaprd,snapdc,snapin,snaplt) busy=(snaprd,snapdc,snapin,snaplt)---------------" );
407446
long next = start+1000L*1000L*1000L;
408447
for(;;) {
409448
ulong snaprd_status = FD_VOLATILE_CONST( snaprd_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
410449
ulong snapdc_status = FD_VOLATILE_CONST( snapdc_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
411450
ulong snapin_status = FD_VOLATILE_CONST( snapin_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
451+
ulong snaplt_status = ULONG_MAX;
452+
453+
ulong snaplt_status_sum = 0UL;
454+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
455+
ulong snaplt_status = FD_VOLATILE_CONST( snaplt_metrics[ i ][ MIDX( GAUGE, TILE, STATUS ) ] );
456+
snaplt_status_sum += snaplt_status;
457+
}
458+
if( FD_UNLIKELY( snaplt_status_sum==2UL*snaplt_tile_cnt ) ) snaplt_status = 2UL;
459+
else snaplt_status = snaplt_tile_cnt>0UL ? 1UL : 2UL;
412460

413-
if( FD_UNLIKELY( snaprd_status==2UL && snapdc_status==2UL && snapin_status == 2UL ) ) break;
461+
if( FD_UNLIKELY( snaprd_status==2UL && snapdc_status==2UL && snapin_status == 2UL && snaplt_status==2UL ) ) break;
414462

415463
long cur = fd_log_wallclock();
416464
if( FD_UNLIKELY( cur<next ) ) {
@@ -430,16 +478,31 @@ backtest_cmd_fn( args_t * args FD_PARAM_UNUSED,
430478
ulong snapin_backp = snapin_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ];
431479
ulong snapin_wait = snapin_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_PREFRAG ) ] +
432480
snapin_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ] + snapin_backp;
481+
ulong snaplt_backp = 0UL;
482+
ulong snaplt_wait = 0UL;
433483

434-
ulong acc_cnt = snapin_metrics[ MIDX( GAUGE, SNAPIN, ACCOUNTS_INSERTED ) ];
435-
printf( "bw=%4.0f MB/s backp=(%3.0f%%,%3.0f%%,%3.0f%%) busy=(%3.0f%%,%3.0f%%,%3.0f%%) acc=%3.1f M/s\n",
484+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
485+
snaplt_backp += snaplt_metrics[ i ][ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ];
486+
}
487+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
488+
snaplt_wait += snaplt_metrics[ i ][ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_PREFRAG ) ] +
489+
snaplt_metrics[ i ][ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ] + snaplt_backp;
490+
}
491+
492+
double snaplt_backp_val = snaplt_tile_cnt ? ((double)(snaplt_backp-snaplt_backp_old)*ns_per_tick )/1e7/(double)snaplt_tile_cnt : 0.0;
493+
double snaplt_busy_val = snaplt_tile_cnt ? 100-(((double)(snaplt_wait-snaplt_wait_old)*ns_per_tick)/1e7/(double)snaplt_tile_cnt) : 0.0;
494+
495+
ulong acc_cnt = snapin_metrics[ MIDX( GAUGE, SNAPIN, ACCOUNTS_INSERTED ) ];
496+
printf( "bw=%4.0f MB/s backp=(%3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%) busy=(%3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%) acc=%3.1f M/s\n",
436497
(double)( total_off-total_off_old )/1e6,
437498
( (double)( snaprd_backp-snaprd_backp_old )*ns_per_tick )/1e7,
438499
( (double)( snapdc_backp-snapdc_backp_old )*ns_per_tick )/1e7,
439500
( (double)( snapin_backp-snapin_backp_old )*ns_per_tick )/1e7,
501+
snaplt_backp_val,
440502
100-( ( (double)( snaprd_wait-snaprd_wait_old )*ns_per_tick )/1e7 ),
441503
100-( ( (double)( snapdc_wait-snapdc_wait_old )*ns_per_tick )/1e7 ),
442504
100-( ( (double)( snapin_wait-snapin_wait_old )*ns_per_tick )/1e7 ),
505+
snaplt_busy_val,
443506
(double)( acc_cnt-acc_cnt_old )/1e6 );
444507
fflush( stdout );
445508
total_off_old = total_off;
@@ -449,12 +512,13 @@ backtest_cmd_fn( args_t * args FD_PARAM_UNUSED,
449512
snapdc_wait_old = snapdc_wait;
450513
snapin_backp_old = snapin_backp;
451514
snapin_wait_old = snapin_wait;
515+
snaplt_backp_old = snaplt_backp;
516+
snaplt_wait_old = snaplt_wait;
452517
acc_cnt_old = acc_cnt;
453518

454519
next+=1000L*1000L*1000L;
455520
}
456521
}
457-
458522
for(;;) pause();
459523
}
460524

‎src/app/firedancer-dev/commands/snapshot_load.c‎

Lines changed: 70 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include "../../../disco/metrics/fd_metrics.h"
77
#include "../../../disco/topo/fd_topob.h"
88
#include "../../../util/tile/fd_tile_private.h"
9+
#include "../../../discof/restore/utils/fd_ssctrl.h"
910
#include "../../../discof/restore/utils/fd_ssmsg.h"
1011

1112
#include <sys/resource.h>
@@ -25,7 +26,9 @@ snapshot_load_topo( config_t * config,
2526
args_t const * args ) {
2627
fd_topo_t * topo = &config->topo;
2728
fd_topob_new( &config->topo, config->name );
28-
topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
29+
topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
30+
ulong snaplt_tile_cnt = config->firedancer.layout.snaplt_tile_count;
31+
int snaplt_disabled = config->development.snapshots.disable_lthash_verification;
2932

3033
fd_topob_wksp( topo, "funk" );
3134
fd_topo_obj_t * funk_obj = setup_topo_funk( topo, "funk",
@@ -37,7 +40,7 @@ snapshot_load_topo( config_t * config,
3740
static ushort tile_to_cpu[ FD_TILE_MAX ] = {0};
3841
if( args->snapshot_load.tile_cpus[0] ) {
3942
ulong cpu_cnt = fd_tile_private_cpus_parse( args->snapshot_load.tile_cpus, tile_to_cpu );
40-
if( FD_UNLIKELY( cpu_cnt<4UL ) ) FD_LOG_ERR(( "--tile-cpus specifies %lu CPUs, but need at least 4", cpu_cnt ));
43+
if( FD_UNLIKELY( cpu_cnt<4UL + (snaplt_disabled?0:snaplt_tile_cnt) ) ) FD_LOG_ERR(( "--tile-cpus specifies %lu CPUs, but need at least %lu", cpu_cnt, 4UL + (snaplt_disabled?snaplt_tile_cnt:0) ));
4144
}
4245

4346
/* metrics tile *****************************************************/
@@ -77,6 +80,23 @@ snapshot_load_topo( config_t * config,
7780
fd_topo_tile_t * snapin_tile = fd_topob_tile( topo, "snapin", "snapin", "snapin", tile_to_cpu[3], 0, 0 );
7881
snapin_tile->allow_shutdown = 1;
7982

83+
if( FD_LIKELY( !snaplt_disabled ) ) {
84+
fd_topob_wksp( topo, "snaplt" );
85+
fd_topob_wksp( topo, "snapin_lt" );
86+
fd_topob_wksp( topo, "snaplt_out" );
87+
88+
#define FOR(cnt) for( ulong i=0UL; i<cnt; i++ )
89+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
90+
fd_topo_tile_t * snaplt_tile = fd_topob_tile( topo, "snaplt", "snaplt", "metric_in", tile_to_cpu[4 + i], 0, 0 );
91+
snaplt_tile->allow_shutdown = 1;
92+
}
93+
}
94+
95+
if( FD_LIKELY( !snaplt_disabled ) ) {
96+
fd_topob_link( topo, "snapin_lt", "snapin_lt", 128UL, sizeof(fd_snapshot_existing_account_t), 1UL );
97+
FOR(snaplt_tile_cnt) fd_topob_link( topo, "snaplt_out", "snaplt_out", 128UL, 2048UL, 1UL );
98+
}
99+
80100
/* uncompressed stream -> snapin tile */
81101
fd_topob_tile_in ( topo, "snapin", 0UL, "metric_in", "snap_stream", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
82102

@@ -99,6 +119,13 @@ snapshot_load_topo( config_t * config,
99119
fd_topob_tile_in( topo, "snaprd", 0UL, "metric_in", "snapin_rd", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
100120
fd_topob_tile_out( topo, "snapin", 0UL, "snapin_rd", 0UL );
101121

122+
if( FD_LIKELY( !snaplt_disabled ) ) {
123+
fd_topob_tile_out( topo, "snapin", 0UL, "snapin_lt", 0UL );
124+
FOR(snaplt_tile_cnt) fd_topob_tile_in( topo, "snapin", 0UL, "metric_in", "snaplt_out", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
125+
FOR(snaplt_tile_cnt) fd_topob_tile_in( topo, "snaplt", i, "metric_in", "snapin_lt", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
126+
FOR(snaplt_tile_cnt) fd_topob_tile_out( topo, "snaplt", i, "snaplt_out", i );
127+
}
128+
102129
for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
103130
fd_topo_tile_t * tile = &topo->tiles[ i ];
104131
fd_topo_configure_tile( tile, config );
@@ -159,6 +186,16 @@ snapshot_load_cmd_fn( args_t * args,
159186
fd_topo_tile_t * snapdc_tile = &topo->tiles[ fd_topo_find_tile( topo, "snapdc", 0UL ) ];
160187
fd_topo_tile_t * snapin_tile = &topo->tiles[ fd_topo_find_tile( topo, "snapin", 0UL ) ];
161188

189+
ulong volatile * snaplt_metrics[ FD_MAX_SNAPLT_TILES ];
190+
ulong snaplt_tile_cnt = fd_topo_tile_name_cnt( topo, "snaplt" );
191+
192+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
193+
ulong snaplt_tile_idx = fd_topo_find_tile( topo, "snaplt", i );
194+
FD_TEST( snaplt_tile_idx!=ULONG_MAX );
195+
fd_topo_tile_t * snaplt_tile = &topo->tiles[ snaplt_tile_idx ];
196+
snaplt_metrics[ i ] = fd_metrics_tile( snaplt_tile->metrics );
197+
}
198+
162199
ulong volatile * const snaprd_metrics = fd_metrics_tile( snaprd_tile->metrics );
163200
ulong volatile * const snapdc_metrics = fd_metrics_tile( snapdc_tile->metrics );
164201
ulong volatile * const snapin_metrics = fd_metrics_tile( snapin_tile->metrics );
@@ -170,6 +207,8 @@ snapshot_load_cmd_fn( args_t * args,
170207
ulong snapdc_wait_old = 0UL;
171208
ulong snapin_backp_old = 0UL;
172209
ulong snapin_wait_old = 0UL;
210+
ulong snaplt_backp_old = 0UL;
211+
ulong snaplt_wait_old = 0UL;
173212
ulong acc_cnt_old = 0UL;
174213
sleep( 1 );
175214
puts( "" );
@@ -179,14 +218,23 @@ snapshot_load_cmd_fn( args_t * args,
179218
puts( "- stall: Waiting on upstream tile" );
180219
puts( "- acc: Number of accounts" );
181220
puts( "" );
182-
puts( "-------------backp=(snaprd,snapdc,snapin) busy=(snaprd,snapdc,snapin)---------------" );
221+
puts( "-------------backp=(snaprd,snapdc,snapin,snaplt) busy=(snaprd,snapdc,snapin,snaplt)---------------" );
183222
long next = start+1000L*1000L*1000L;
184223
for(;;) {
185224
ulong snaprd_status = FD_VOLATILE_CONST( snaprd_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
186225
ulong snapdc_status = FD_VOLATILE_CONST( snapdc_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
187226
ulong snapin_status = FD_VOLATILE_CONST( snapin_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
227+
ulong snaplt_status = ULONG_MAX;
228+
229+
ulong snaplt_status_sum = 0UL;
230+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
231+
ulong snaplt_status = FD_VOLATILE_CONST( snaplt_metrics[ i ][ MIDX( GAUGE, TILE, STATUS ) ] );
232+
snaplt_status_sum += snaplt_status;
233+
}
234+
if( FD_UNLIKELY( snaplt_status_sum==2UL*snaplt_tile_cnt ) ) snaplt_status = 2UL;
235+
else snaplt_status = snaplt_tile_cnt>0UL ? 1UL : 2UL;
188236

189-
if( FD_UNLIKELY( snaprd_status==2UL && snapdc_status==2UL && snapin_status == 2UL ) ) break;
237+
if( FD_UNLIKELY( snaprd_status==2UL && snapdc_status==2UL && snapin_status == 2UL && snaplt_status==2UL ) ) break;
190238

191239
long cur = fd_log_wallclock();
192240
if( FD_UNLIKELY( cur<next ) ) {
@@ -203,19 +251,32 @@ snapshot_load_cmd_fn( args_t * args,
203251
ulong snapdc_wait = snapdc_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ] + snapdc_backp;
204252
ulong snapin_backp = snapin_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ];
205253
ulong snapin_wait = snapin_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ] + snapin_backp;
254+
ulong snaplt_backp = 0UL;
255+
ulong snaplt_wait = 0UL;
256+
257+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
258+
snaplt_backp += snaplt_metrics[ i ][ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ];
259+
}
260+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
261+
snaplt_wait += snaplt_metrics[ i ][ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ] + snaplt_backp;
262+
}
206263

207-
double progress = 100.0 * (double)snaprd_metrics[ MIDX( GAUGE, SNAPRD, FULL_BYTES_READ ) ] / (double)snaprd_metrics[ MIDX( GAUGE, SNAPRD, FULL_BYTES_TOTAL ) ];
264+
double progress = 100.0 * (double)snaprd_metrics[ MIDX( GAUGE, SNAPRD, FULL_BYTES_READ ) ] / (double)snaprd_metrics[ MIDX( GAUGE, SNAPRD, FULL_BYTES_TOTAL ) ];
265+
double snaplt_backp_val = snaplt_tile_cnt ? ((double)(snaplt_backp-snaplt_backp_old)*ns_per_tick)/1e7/(double)snaplt_tile_cnt : 0.0;
266+
double snaplt_busy_val = snaplt_tile_cnt ? 100-(((double)( snaplt_wait-snaplt_wait_old)*ns_per_tick)/1e7/(double)snaplt_tile_cnt) : 0.0;
267+
ulong acc_cnt = snapin_metrics[ MIDX( GAUGE, SNAPIN, ACCOUNTS_INSERTED ) ];
208268

209-
ulong acc_cnt = snapin_metrics[ MIDX( GAUGE, SNAPIN, ACCOUNTS_INSERTED ) ];
210-
printf( "%.1f %% bw=%4.0f MB/s backp=(%3.0f%%,%3.0f%%,%3.0f%%) busy=(%3.0f%%,%3.0f%%,%3.0f%%) acc=%3.1f M/s\n",
269+
printf( "%.1f %% bw=%4.0f MB/s backp=(%3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%) busy=(%3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%) acc=%3.1f M/s\n",
211270
progress,
212271
(double)( total_off-total_off_old )/1e6,
213272
( (double)( snaprd_backp-snaprd_backp_old )*ns_per_tick )/1e7,
214273
( (double)( snapdc_backp-snapdc_backp_old )*ns_per_tick )/1e7,
215274
( (double)( snapin_backp-snapin_backp_old )*ns_per_tick )/1e7,
275+
snaplt_backp_val,
216276
100-( ( (double)( snaprd_wait-snaprd_wait_old )*ns_per_tick )/1e7 ),
217277
100-( ( (double)( snapdc_wait-snapdc_wait_old )*ns_per_tick )/1e7 ),
218278
100-( ( (double)( snapin_wait-snapin_wait_old )*ns_per_tick )/1e7 ),
279+
snaplt_busy_val,
219280
(double)( acc_cnt-acc_cnt_old )/1e6 );
220281
fflush( stdout );
221282
total_off_old = total_off;
@@ -225,6 +286,8 @@ snapshot_load_cmd_fn( args_t * args,
225286
snapdc_wait_old = snapdc_wait;
226287
snapin_backp_old = snapin_backp;
227288
snapin_wait_old = snapin_wait;
289+
snaplt_backp_old = snaplt_backp;
290+
snaplt_wait_old = snaplt_wait;
228291
acc_cnt_old = acc_cnt;
229292

230293
next+=1000L*1000L*1000L;

0 commit comments

Comments
 (0)