replay: improve reliability of replay to tower vote states sending

src/discof/replay/fd_replay_tile.c

@@ -549,10 +549,10 @@ publish_stake_weights( fd_replay_tile_t *   ctx,
    - stake:          The stake amount associated with this vote account
    - vote_tower_out: Output structure to populate with vote state information
 
-   Failure modes:
-   - Vote account data is too large (returns -1)
-   - Vote account is not found in Funk (returns -1) */
-static int
+   This function should never fail, as the vote_states cache is guarenteed to
+   only contain valid vote accounts.
+*/
+static void
 fd_replay_out_vote_tower_from_funk(
     fd_accdb_user_t *         accdb,
     fd_funk_txn_xid_t const * xid,
@@ -571,18 +571,15 @@ fd_replay_out_vote_tower_from_funk(
 
     fd_accdb_peek_t peek[1];
     if( FD_UNLIKELY( !fd_accdb_peek( accdb, peek, xid, pubkey->uc ) ) ) {
-      /* FIXME crash here? */
-      FD_LOG_WARNING(( "vote account not found. address: %s", FD_BASE58_ENC_32_ALLOCA( pubkey->uc ) ));
-      return -1;
+      FD_LOG_CRIT(( "vote account not found. address: %s", FD_BASE58_ENC_32_ALLOCA( pubkey->uc ) ));
     }
 
     ulong data_sz = fd_accdb_ref_data_sz( peek->acc );
     if( FD_UNLIKELY( data_sz > sizeof(vote_tower_out->acc) ) ) {
-      FD_LOG_WARNING(( "vote account %s has too large data. dlen %lu > %lu",
+      FD_LOG_CRIT(( "vote account %s has too large data. dlen %lu > %lu",
         FD_BASE58_ENC_32_ALLOCA( pubkey->uc ),
         data_sz,
         sizeof(vote_tower_out->acc) ));
-      return -1;
     }
 
     fd_memcpy( vote_tower_out->acc, fd_accdb_ref_data_const( peek->acc ), data_sz );
@@ -591,8 +588,6 @@ fd_replay_out_vote_tower_from_funk(
     if( FD_LIKELY( fd_accdb_peek_test( peek ) ) ) break;
     FD_SPIN_PAUSE();
   }
-
-  return 0;
 }
 
 /* This function buffers all the vote account towers that Tower needs at
@@ -608,7 +603,7 @@ buffer_vote_towers( fd_replay_tile_t *        ctx,
   ctx->vote_tower_out_idx = 0UL;
   ctx->vote_tower_out_len = 0UL;
 
-  fd_vote_states_t const * vote_states = fd_bank_vote_states_prev_locking_query( bank );
+  fd_vote_states_t const * vote_states = fd_bank_vote_states_locking_query( bank );
   fd_vote_states_iter_t iter_[1];
   for( fd_vote_states_iter_t * iter = fd_vote_states_iter_init( iter_, vote_states );
        !fd_vote_states_iter_done( iter );
@@ -617,15 +612,14 @@ buffer_vote_towers( fd_replay_tile_t *        ctx,
     if( FD_UNLIKELY( vote_state->stake == 0 ) ) continue; /* skip unstaked vote accounts */
     fd_pubkey_t const * vote_account_pubkey = &vote_state->vote_account;
     if( FD_UNLIKELY( ctx->vote_tower_out_len >= (FD_REPLAY_TOWER_VOTE_ACC_MAX-1UL) ) ) FD_LOG_ERR(( "vote_tower_out_len too large" ));
-    if( FD_UNLIKELY( fd_replay_out_vote_tower_from_funk( ctx->accdb,
-                                                         xid,
-                                                         vote_account_pubkey,
-                                                         vote_state->stake,
-                                                         &ctx->vote_tower_out[ctx->vote_tower_out_len++] ) ) ) {
-      FD_LOG_DEBUG(( "failed to get vote state for vote account %s", FD_BASE58_ENC_32_ALLOCA( vote_account_pubkey->uc ) ));
-    }
+    fd_replay_out_vote_tower_from_funk(
+      ctx->accdb,
+      xid,
+      vote_account_pubkey,
+      vote_state->stake_t_2,
+      &ctx->vote_tower_out[ ctx->vote_tower_out_len++ ] );
   }
-  fd_bank_vote_states_prev_end_locking_query( bank );
+  fd_bank_vote_states_end_locking_query( bank );
 }
 
 /* This function publishes the next vote tower in the

src/discof/restore/utils/fd_ssload.c

@@ -270,13 +270,17 @@ fd_ssload_recover( fd_snapshot_manifest_t *  manifest,
     fd_vote_state_ele_t * vote_state_prev_prev = fd_vote_states_query( vote_stakes_prev_prev, (fd_pubkey_t *)elem->vote_account_pubkey );
     ulong prev_prev_stake = vote_state_prev_prev ? vote_state_prev_prev->stake : 0UL;
 
+    fd_vote_state_ele_t * vote_state_prev = fd_vote_states_query( vote_stakes_prev, (fd_pubkey_t *)elem->vote_account_pubkey );
+    ulong prev_stake = vote_state_prev ? vote_state_prev->stake : 0UL;
+
     fd_vote_state_ele_t * vote_state = fd_vote_states_update( vote_states, (fd_pubkey_t *)elem->vote_account_pubkey );
 
     vote_state->node_account        = *(fd_pubkey_t *)elem->node_account_pubkey;
     vote_state->commission          = elem->commission;
     vote_state->last_vote_timestamp = elem->last_timestamp;
     vote_state->last_vote_slot      = elem->last_slot;
     vote_state->stake               = elem->stake;
+    vote_state->stake_t_1           = prev_stake;
     vote_state->stake_t_2           = prev_prev_stake;
   }
   fd_bank_vote_states_end_locking_modify( bank );

src/flamenco/runtime/fd_runtime.c

@@ -1565,6 +1565,7 @@ fd_runtime_init_bank_from_genesis( fd_banks_t *                       banks,
     if( !memcmp( acc->account.owner.key, fd_solana_vote_program_id.key, sizeof(fd_pubkey_t) ) ) {
       fd_vote_state_ele_t * vote_state = fd_vote_states_query( vote_states, &acc->key );
 
+      vote_state->stake_t_1 = vote_state->stake;
       vote_state->stake_t_2 = vote_state->stake;
     }
   }

src/flamenco/runtime/tests/fd_block_harness.c

@@ -35,6 +35,7 @@ typedef struct {
    using the stake delegations cache. */
 static void
 fd_solfuzz_block_refresh_vote_accounts( fd_vote_states_t *       vote_states,
+                                        fd_vote_states_t *       vote_states_prev,
                                         fd_vote_states_t *       vote_states_prev_prev,
                                         fd_stake_delegations_t * stake_delegations,
                                         ulong                    epoch ) {
@@ -53,6 +54,7 @@ fd_solfuzz_block_refresh_vote_accounts( fd_vote_states_t *       vote_states,
     if( !vote_state ) continue;
 
     vote_state->stake     += stake;
+    vote_state->stake_t_1 += stake;
     vote_state->stake_t_2 += stake;
   }
 
@@ -70,6 +72,17 @@ fd_solfuzz_block_refresh_vote_accounts( fd_vote_states_t *       vote_states,
     vote_state->stake_t_2 = epoch>=2UL ? t_2_stake : vote_state->stake;
     vote_state->stake_t_2 = vote_state->stake;
   }
+
+  /* Set stake_t_1 for the vote accounts in the vote states cache. */
+  for( fd_vote_states_iter_t * iter = fd_vote_states_iter_init( vs_iter_, vote_states_prev );
+       !fd_vote_states_iter_done( iter );
+       fd_vote_states_iter_next( iter ) ) {
+    fd_vote_state_ele_t * vote_state = fd_vote_states_iter_ele( iter );
+    fd_vote_state_ele_t * vote_state_prev = fd_vote_states_query( vote_states_prev, &vote_state->vote_account );
+    ulong t_1_stake = !!vote_state_prev ? vote_state_prev->stake : 0UL;
+    vote_state->stake_t_1 = epoch>=1UL ? t_1_stake : vote_state->stake;
+    vote_state->stake_t_1 = vote_state->stake;
+  }
 }
 
 /* Registers a single vote account into the current votes cache.  The
@@ -180,6 +193,7 @@ fd_solfuzz_pb_block_update_prev_epoch_votes_cache( fd_vote_states_t *
       fd_vote_states_update_from_account( vote_states, &vote_address, vote_data, vote_data_len );
       fd_vote_state_ele_t * vote_state = fd_vote_states_query( vote_states, &vote_address );
       vote_state->stake     += stake;
+      vote_state->stake_t_1 += stake;
       vote_state->stake_t_2 += stake;
     }
   } FD_SPAD_FRAME_END;
@@ -347,7 +361,12 @@ fd_solfuzz_pb_block_ctx_create( fd_solfuzz_runner_t *                runner,
       runner->spad );
 
   /* Refresh vote accounts to calculate stake delegations */
-  fd_solfuzz_block_refresh_vote_accounts( vote_states, vote_states_prev_prev, stake_delegations, fd_bank_epoch_get( bank ) );
+  fd_solfuzz_block_refresh_vote_accounts(
+    vote_states,
+    vote_states_prev,
+    vote_states_prev_prev,
+    stake_delegations,
+    fd_bank_epoch_get( bank ) );
   fd_bank_vote_states_end_locking_modify( bank );
 
   fd_bank_vote_states_prev_prev_end_locking_modify( bank );
@@ -639,7 +658,7 @@ fd_solfuzz_pb_build_leader_schedule_effects( fd_solfuzz_runner_t *
       memcpy( &pubkey, test_ctx->acct_states[i].address, sizeof(fd_pubkey_t) );
       fd_solfuzz_block_register_stake_delegation( runner->accdb->funk, xid, stake_delegations, &pubkey );
     }
-    fd_solfuzz_block_refresh_vote_accounts( tmp_vs, tmp_vs, stake_delegations, fd_bank_epoch_get( runner->bank ) );
+    fd_solfuzz_block_refresh_vote_accounts( tmp_vs, tmp_vs, tmp_vs, stake_delegations, fd_bank_epoch_get( runner->bank ) );
   }
 
   /* Build weights from the selected stake source */

src/flamenco/stakes/fd_stakes.c

@@ -84,19 +84,33 @@ fd_refresh_vote_accounts( fd_bank_t *                    bank,
      to make clock calulations more efficient.  This is purely an
      optimization. */
 
-  fd_vote_states_t const * vote_states_prev = fd_bank_vote_states_prev_prev_locking_query( bank );
+  fd_vote_states_t const * vote_states_prev_prev = fd_bank_vote_states_prev_prev_locking_query( bank );
+  if( FD_LIKELY( fd_bank_slot_get( bank )!=0UL ) ) {
+    fd_vote_states_iter_t vs_iter_[1];
+    for( fd_vote_states_iter_t * vs_iter = fd_vote_states_iter_init( vs_iter_, vote_states );
+        !fd_vote_states_iter_done( vs_iter );
+        fd_vote_states_iter_next( vs_iter ) ) {
+      fd_vote_state_ele_t * vote_state           = fd_vote_states_iter_ele( vs_iter );
+      fd_vote_state_ele_t * vote_state_prev_prev = fd_vote_states_query( vote_states_prev_prev, &vote_state->vote_account );
+      vote_state->stake_t_2 = !!vote_state_prev_prev ? vote_state_prev_prev->stake : 0UL;
+    }
+  }
+  fd_bank_vote_states_prev_prev_end_locking_query( bank );
 
+  /* Cache the stake from epoch T-1 for the vote accounts in the vote
+     states cache, to be sent to Tower for it's threshold checks. */
+  fd_vote_states_t const * vote_states_prev = fd_bank_vote_states_prev_locking_query( bank );
   if( FD_LIKELY( fd_bank_slot_get( bank )!=0UL ) ) {
     fd_vote_states_iter_t vs_iter_[1];
     for( fd_vote_states_iter_t * vs_iter = fd_vote_states_iter_init( vs_iter_, vote_states );
         !fd_vote_states_iter_done( vs_iter );
         fd_vote_states_iter_next( vs_iter ) ) {
       fd_vote_state_ele_t * vote_state      = fd_vote_states_iter_ele( vs_iter );
       fd_vote_state_ele_t * vote_state_prev = fd_vote_states_query( vote_states_prev, &vote_state->vote_account );
-      vote_state->stake_t_2 = !!vote_state_prev ? vote_state_prev->stake : 0UL;
+      vote_state->stake_t_1 = !!vote_state_prev ? vote_state_prev->stake : 0UL;
     }
   }
-  fd_bank_vote_states_prev_prev_end_locking_query( bank );
+  fd_bank_vote_states_prev_end_locking_query( bank );
   fd_bank_vote_states_end_locking_modify( bank );
 }
 

src/flamenco/stakes/fd_vote_states.c

@@ -199,6 +199,7 @@ fd_vote_states_update( fd_vote_states_t *  vote_states,
 
   vote_state->vote_account = *vote_account;
   vote_state->stake        = 0UL;
+  vote_state->stake_t_1    = 0UL;
   vote_state->stake_t_2    = 0UL;
 
   if( FD_UNLIKELY( !fd_vote_state_map_ele_insert(
@@ -330,6 +331,7 @@ fd_vote_states_reset_stakes( fd_vote_states_t * vote_states ) {
     }
 
     vote_state->stake     = 0UL;
+    vote_state->stake_t_1 = 0UL;
     vote_state->stake_t_2 = 0UL;
   }
 }

src/flamenco/stakes/fd_vote_states.h

@@ -52,6 +52,7 @@
    - stake: stake as of the end of the previous epoch.  This is used
      eventually for leader schedule calculations.  The stake from epoch
      T-2 (stake_t_2) is used for the stake in clock calculations.
+     T-1 (stake_t_1) is used for the stake in Tower's threshold switch checks.
    - rewards: this information is only used at the epoch boundary.
 */
 
@@ -76,8 +77,11 @@ struct fd_vote_state_ele {
 
   /* Vote account stake information which is derived from the stake
      delegations.  This information is used for leader schedule
-     calculation and clock stake-weighted median calculations. */
+     calculation and clock stake-weighted median calculations.
+
+     stake_t_1 is used in Tower, for it's threshold switch checks. */
   ulong       stake;
+  ulong       stake_t_1;
   ulong       stake_t_2;
 
   /* Vote account information which is derived from the vote account