From 796172317b2f22705644cb32c64774d78113b3f0 Mon Sep 17 00:00:00 2001 From: Tom Pointon Date: Fri, 7 Nov 2025 15:25:46 +0000 Subject: [PATCH] replay: improve reliability of replay to tower vote states sending --- src/discof/replay/fd_replay_tile.c | 34 ++++++++----------- src/discof/restore/utils/fd_ssload.c | 4 +++ src/flamenco/runtime/fd_runtime.c | 1 + src/flamenco/runtime/tests/fd_block_harness.c | 23 +++++++++++-- src/flamenco/stakes/fd_stakes.c | 20 +++++++++-- src/flamenco/stakes/fd_vote_states.c | 2 ++ src/flamenco/stakes/fd_vote_states.h | 6 +++- 7 files changed, 64 insertions(+), 26 deletions(-) diff --git a/src/discof/replay/fd_replay_tile.c b/src/discof/replay/fd_replay_tile.c index 25d24109b5a..fa0c15d7c4c 100644 --- a/src/discof/replay/fd_replay_tile.c +++ b/src/discof/replay/fd_replay_tile.c @@ -549,10 +549,10 @@ publish_stake_weights( fd_replay_tile_t * ctx, - stake: The stake amount associated with this vote account - vote_tower_out: Output structure to populate with vote state information - Failure modes: - - Vote account data is too large (returns -1) - - Vote account is not found in Funk (returns -1) */ -static int + This function should never fail, as the vote_states cache is guarenteed to + only contain valid vote accounts. +*/ +static void fd_replay_out_vote_tower_from_funk( fd_accdb_user_t * accdb, fd_funk_txn_xid_t const * xid, @@ -571,18 +571,15 @@ fd_replay_out_vote_tower_from_funk( fd_accdb_peek_t peek[1]; if( FD_UNLIKELY( !fd_accdb_peek( accdb, peek, xid, pubkey->uc ) ) ) { - /* FIXME crash here? */ - FD_LOG_WARNING(( "vote account not found. address: %s", FD_BASE58_ENC_32_ALLOCA( pubkey->uc ) )); - return -1; + FD_LOG_CRIT(( "vote account not found. address: %s", FD_BASE58_ENC_32_ALLOCA( pubkey->uc ) )); } ulong data_sz = fd_accdb_ref_data_sz( peek->acc ); if( FD_UNLIKELY( data_sz > sizeof(vote_tower_out->acc) ) ) { - FD_LOG_WARNING(( "vote account %s has too large data. dlen %lu > %lu", + FD_LOG_CRIT(( "vote account %s has too large data. dlen %lu > %lu", FD_BASE58_ENC_32_ALLOCA( pubkey->uc ), data_sz, sizeof(vote_tower_out->acc) )); - return -1; } fd_memcpy( vote_tower_out->acc, fd_accdb_ref_data_const( peek->acc ), data_sz ); @@ -591,8 +588,6 @@ fd_replay_out_vote_tower_from_funk( if( FD_LIKELY( fd_accdb_peek_test( peek ) ) ) break; FD_SPIN_PAUSE(); } - - return 0; } /* This function buffers all the vote account towers that Tower needs at @@ -608,7 +603,7 @@ buffer_vote_towers( fd_replay_tile_t * ctx, ctx->vote_tower_out_idx = 0UL; ctx->vote_tower_out_len = 0UL; - fd_vote_states_t const * vote_states = fd_bank_vote_states_prev_locking_query( bank ); + fd_vote_states_t const * vote_states = fd_bank_vote_states_locking_query( bank ); fd_vote_states_iter_t iter_[1]; for( fd_vote_states_iter_t * iter = fd_vote_states_iter_init( iter_, vote_states ); !fd_vote_states_iter_done( iter ); @@ -617,15 +612,14 @@ buffer_vote_towers( fd_replay_tile_t * ctx, if( FD_UNLIKELY( vote_state->stake == 0 ) ) continue; /* skip unstaked vote accounts */ fd_pubkey_t const * vote_account_pubkey = &vote_state->vote_account; if( FD_UNLIKELY( ctx->vote_tower_out_len >= (FD_REPLAY_TOWER_VOTE_ACC_MAX-1UL) ) ) FD_LOG_ERR(( "vote_tower_out_len too large" )); - if( FD_UNLIKELY( fd_replay_out_vote_tower_from_funk( ctx->accdb, - xid, - vote_account_pubkey, - vote_state->stake, - &ctx->vote_tower_out[ctx->vote_tower_out_len++] ) ) ) { - FD_LOG_DEBUG(( "failed to get vote state for vote account %s", FD_BASE58_ENC_32_ALLOCA( vote_account_pubkey->uc ) )); - } + fd_replay_out_vote_tower_from_funk( + ctx->accdb, + xid, + vote_account_pubkey, + vote_state->stake_t_2, + &ctx->vote_tower_out[ ctx->vote_tower_out_len++ ] ); } - fd_bank_vote_states_prev_end_locking_query( bank ); + fd_bank_vote_states_end_locking_query( bank ); } /* This function publishes the next vote tower in the diff --git a/src/discof/restore/utils/fd_ssload.c b/src/discof/restore/utils/fd_ssload.c index ca4edf4cc24..8c334dee732 100644 --- a/src/discof/restore/utils/fd_ssload.c +++ b/src/discof/restore/utils/fd_ssload.c @@ -270,6 +270,9 @@ fd_ssload_recover( fd_snapshot_manifest_t * manifest, fd_vote_state_ele_t * vote_state_prev_prev = fd_vote_states_query( vote_stakes_prev_prev, (fd_pubkey_t *)elem->vote_account_pubkey ); ulong prev_prev_stake = vote_state_prev_prev ? vote_state_prev_prev->stake : 0UL; + fd_vote_state_ele_t * vote_state_prev = fd_vote_states_query( vote_stakes_prev, (fd_pubkey_t *)elem->vote_account_pubkey ); + ulong prev_stake = vote_state_prev ? vote_state_prev->stake : 0UL; + fd_vote_state_ele_t * vote_state = fd_vote_states_update( vote_states, (fd_pubkey_t *)elem->vote_account_pubkey ); vote_state->node_account = *(fd_pubkey_t *)elem->node_account_pubkey; @@ -277,6 +280,7 @@ fd_ssload_recover( fd_snapshot_manifest_t * manifest, vote_state->last_vote_timestamp = elem->last_timestamp; vote_state->last_vote_slot = elem->last_slot; vote_state->stake = elem->stake; + vote_state->stake_t_1 = prev_stake; vote_state->stake_t_2 = prev_prev_stake; } fd_bank_vote_states_end_locking_modify( bank ); diff --git a/src/flamenco/runtime/fd_runtime.c b/src/flamenco/runtime/fd_runtime.c index 013e7f41e26..50716288518 100644 --- a/src/flamenco/runtime/fd_runtime.c +++ b/src/flamenco/runtime/fd_runtime.c @@ -1565,6 +1565,7 @@ fd_runtime_init_bank_from_genesis( fd_banks_t * banks, if( !memcmp( acc->account.owner.key, fd_solana_vote_program_id.key, sizeof(fd_pubkey_t) ) ) { fd_vote_state_ele_t * vote_state = fd_vote_states_query( vote_states, &acc->key ); + vote_state->stake_t_1 = vote_state->stake; vote_state->stake_t_2 = vote_state->stake; } } diff --git a/src/flamenco/runtime/tests/fd_block_harness.c b/src/flamenco/runtime/tests/fd_block_harness.c index 678d34920fb..ee05c327931 100644 --- a/src/flamenco/runtime/tests/fd_block_harness.c +++ b/src/flamenco/runtime/tests/fd_block_harness.c @@ -35,6 +35,7 @@ typedef struct { using the stake delegations cache. */ static void fd_solfuzz_block_refresh_vote_accounts( fd_vote_states_t * vote_states, + fd_vote_states_t * vote_states_prev, fd_vote_states_t * vote_states_prev_prev, fd_stake_delegations_t * stake_delegations, ulong epoch ) { @@ -53,6 +54,7 @@ fd_solfuzz_block_refresh_vote_accounts( fd_vote_states_t * vote_states, if( !vote_state ) continue; vote_state->stake += stake; + vote_state->stake_t_1 += stake; vote_state->stake_t_2 += stake; } @@ -70,6 +72,17 @@ fd_solfuzz_block_refresh_vote_accounts( fd_vote_states_t * vote_states, vote_state->stake_t_2 = epoch>=2UL ? t_2_stake : vote_state->stake; vote_state->stake_t_2 = vote_state->stake; } + + /* Set stake_t_1 for the vote accounts in the vote states cache. */ + for( fd_vote_states_iter_t * iter = fd_vote_states_iter_init( vs_iter_, vote_states_prev ); + !fd_vote_states_iter_done( iter ); + fd_vote_states_iter_next( iter ) ) { + fd_vote_state_ele_t * vote_state = fd_vote_states_iter_ele( iter ); + fd_vote_state_ele_t * vote_state_prev = fd_vote_states_query( vote_states_prev, &vote_state->vote_account ); + ulong t_1_stake = !!vote_state_prev ? vote_state_prev->stake : 0UL; + vote_state->stake_t_1 = epoch>=1UL ? t_1_stake : vote_state->stake; + vote_state->stake_t_1 = vote_state->stake; + } } /* Registers a single vote account into the current votes cache. The @@ -180,6 +193,7 @@ fd_solfuzz_pb_block_update_prev_epoch_votes_cache( fd_vote_states_t * fd_vote_states_update_from_account( vote_states, &vote_address, vote_data, vote_data_len ); fd_vote_state_ele_t * vote_state = fd_vote_states_query( vote_states, &vote_address ); vote_state->stake += stake; + vote_state->stake_t_1 += stake; vote_state->stake_t_2 += stake; } } FD_SPAD_FRAME_END; @@ -347,7 +361,12 @@ fd_solfuzz_pb_block_ctx_create( fd_solfuzz_runner_t * runner, runner->spad ); /* Refresh vote accounts to calculate stake delegations */ - fd_solfuzz_block_refresh_vote_accounts( vote_states, vote_states_prev_prev, stake_delegations, fd_bank_epoch_get( bank ) ); + fd_solfuzz_block_refresh_vote_accounts( + vote_states, + vote_states_prev, + vote_states_prev_prev, + stake_delegations, + fd_bank_epoch_get( bank ) ); fd_bank_vote_states_end_locking_modify( bank ); fd_bank_vote_states_prev_prev_end_locking_modify( bank ); @@ -639,7 +658,7 @@ fd_solfuzz_pb_build_leader_schedule_effects( fd_solfuzz_runner_t * memcpy( &pubkey, test_ctx->acct_states[i].address, sizeof(fd_pubkey_t) ); fd_solfuzz_block_register_stake_delegation( runner->accdb->funk, xid, stake_delegations, &pubkey ); } - fd_solfuzz_block_refresh_vote_accounts( tmp_vs, tmp_vs, stake_delegations, fd_bank_epoch_get( runner->bank ) ); + fd_solfuzz_block_refresh_vote_accounts( tmp_vs, tmp_vs, tmp_vs, stake_delegations, fd_bank_epoch_get( runner->bank ) ); } /* Build weights from the selected stake source */ diff --git a/src/flamenco/stakes/fd_stakes.c b/src/flamenco/stakes/fd_stakes.c index 5ce2d4b10f1..a63a60db5ac 100644 --- a/src/flamenco/stakes/fd_stakes.c +++ b/src/flamenco/stakes/fd_stakes.c @@ -84,8 +84,22 @@ fd_refresh_vote_accounts( fd_bank_t * bank, to make clock calulations more efficient. This is purely an optimization. */ - fd_vote_states_t const * vote_states_prev = fd_bank_vote_states_prev_prev_locking_query( bank ); + fd_vote_states_t const * vote_states_prev_prev = fd_bank_vote_states_prev_prev_locking_query( bank ); + if( FD_LIKELY( fd_bank_slot_get( bank )!=0UL ) ) { + fd_vote_states_iter_t vs_iter_[1]; + for( fd_vote_states_iter_t * vs_iter = fd_vote_states_iter_init( vs_iter_, vote_states ); + !fd_vote_states_iter_done( vs_iter ); + fd_vote_states_iter_next( vs_iter ) ) { + fd_vote_state_ele_t * vote_state = fd_vote_states_iter_ele( vs_iter ); + fd_vote_state_ele_t * vote_state_prev_prev = fd_vote_states_query( vote_states_prev_prev, &vote_state->vote_account ); + vote_state->stake_t_2 = !!vote_state_prev_prev ? vote_state_prev_prev->stake : 0UL; + } + } + fd_bank_vote_states_prev_prev_end_locking_query( bank ); + /* Cache the stake from epoch T-1 for the vote accounts in the vote + states cache, to be sent to Tower for it's threshold checks. */ + fd_vote_states_t const * vote_states_prev = fd_bank_vote_states_prev_locking_query( bank ); if( FD_LIKELY( fd_bank_slot_get( bank )!=0UL ) ) { fd_vote_states_iter_t vs_iter_[1]; for( fd_vote_states_iter_t * vs_iter = fd_vote_states_iter_init( vs_iter_, vote_states ); @@ -93,10 +107,10 @@ fd_refresh_vote_accounts( fd_bank_t * bank, fd_vote_states_iter_next( vs_iter ) ) { fd_vote_state_ele_t * vote_state = fd_vote_states_iter_ele( vs_iter ); fd_vote_state_ele_t * vote_state_prev = fd_vote_states_query( vote_states_prev, &vote_state->vote_account ); - vote_state->stake_t_2 = !!vote_state_prev ? vote_state_prev->stake : 0UL; + vote_state->stake_t_1 = !!vote_state_prev ? vote_state_prev->stake : 0UL; } } - fd_bank_vote_states_prev_prev_end_locking_query( bank ); + fd_bank_vote_states_prev_end_locking_query( bank ); fd_bank_vote_states_end_locking_modify( bank ); } diff --git a/src/flamenco/stakes/fd_vote_states.c b/src/flamenco/stakes/fd_vote_states.c index a2ca51dda44..4ca912d9e75 100644 --- a/src/flamenco/stakes/fd_vote_states.c +++ b/src/flamenco/stakes/fd_vote_states.c @@ -199,6 +199,7 @@ fd_vote_states_update( fd_vote_states_t * vote_states, vote_state->vote_account = *vote_account; vote_state->stake = 0UL; + vote_state->stake_t_1 = 0UL; vote_state->stake_t_2 = 0UL; if( FD_UNLIKELY( !fd_vote_state_map_ele_insert( @@ -330,6 +331,7 @@ fd_vote_states_reset_stakes( fd_vote_states_t * vote_states ) { } vote_state->stake = 0UL; + vote_state->stake_t_1 = 0UL; vote_state->stake_t_2 = 0UL; } } diff --git a/src/flamenco/stakes/fd_vote_states.h b/src/flamenco/stakes/fd_vote_states.h index 7c725e41e81..2f09db496fa 100644 --- a/src/flamenco/stakes/fd_vote_states.h +++ b/src/flamenco/stakes/fd_vote_states.h @@ -52,6 +52,7 @@ - stake: stake as of the end of the previous epoch. This is used eventually for leader schedule calculations. The stake from epoch T-2 (stake_t_2) is used for the stake in clock calculations. + T-1 (stake_t_1) is used for the stake in Tower's threshold switch checks. - rewards: this information is only used at the epoch boundary. */ @@ -76,8 +77,11 @@ struct fd_vote_state_ele { /* Vote account stake information which is derived from the stake delegations. This information is used for leader schedule - calculation and clock stake-weighted median calculations. */ + calculation and clock stake-weighted median calculations. + + stake_t_1 is used in Tower, for it's threshold switch checks. */ ulong stake; + ulong stake_t_1; ulong stake_t_2; /* Vote account information which is derived from the vote account