feat(tower): minor hard fork detector revisions and integration

Author: lidatong

src/app/firedancer/topology.c

@@ -345,7 +345,6 @@ fd_topo_initialize( config_t * config ) {
 
   fd_topob_wksp( topo, "funk"         );
   fd_topob_wksp( topo, "progcache"    );
-  fd_topob_wksp( topo, "bh_cmp"       );
   fd_topob_wksp( topo, "fec_sets"     );
   fd_topob_wksp( topo, "txncache"     );
   fd_topob_wksp( topo, "banks"        );

src/discof/bank/fd_bank_tile.c

@@ -587,7 +587,6 @@ unprivileged_init( fd_topo_t *      topo,
       ctx->txn_ctx[ i ].bundle.prev_txn_ctxs[ j ] = &ctx->txn_ctx[ j ];
     }
 
-    ctx->txn_ctx[ i ].bank_hash_cmp    = NULL; /* TODO - do we need this? */
     ctx->txn_ctx[ i ].progcache        = ctx->txn_ctx[ i ]._progcache;
     ctx->txn_ctx[ i ].status_cache     = txncache;
     *(ctx->txn_ctx[ i ].funk)          = *funk;

src/discof/exec/fd_exec_tile.c

@@ -288,7 +288,6 @@ unprivileged_init( fd_topo_t *      topo,
     FD_LOG_CRIT(( "fd_progcache_join() failed" ));
   }
   ctx->txn_ctx->status_cache     = ctx->txncache;
-  ctx->txn_ctx->bank_hash_cmp    = ctx->bank_hash_cmp;
   ctx->txn_ctx->bundle.is_bundle = 0;
 
   /********************************************************************/

src/discof/tower/fd_hard_fork_detector.c

@@ -86,7 +86,7 @@ typedef struct voted_on voted_on_t;
 
 struct block_result_key {
   fd_pubkey_t block_id;
-  fd_pubkey_t block_hash;
+  fd_pubkey_t bank_hash;
 };
 
 typedef struct block_result_key block_result_key_t;
@@ -126,8 +126,8 @@ typedef struct block_result block_result_t;
 #define MAP_PREV               map.prev
 #define MAP_NEXT               map.next
 #define MAP_KEY_EQ(k0,k1)      ( fd_pubkey_eq( &((k0)->block_id),   &((k1)->block_id)   ) &\
-                                fd_pubkey_eq( &((k0)->block_hash), &((k1)->block_hash) ) )
-#define MAP_KEY_HASH(key,seed) ( seed ^ fd_ulong_load_8( (key)->block_id.uc ) ^ fd_ulong_load_8( (key)->block_hash.uc ) )
+                                fd_pubkey_eq( &((k0)->bank_hash), &((k1)->bank_hash) ) )
+#define MAP_KEY_HASH(key,seed) ( seed ^ fd_ulong_load_8( (key)->block_id.uc ) ^ fd_ulong_load_8( (key)->bank_hash.uc ) )
 #define MAP_OPTIMIZE_RANDOM_ACCESS_REMOVAL 1
 #include "../../util/tmpl/fd_map_chain.c"
 
@@ -139,7 +139,7 @@ typedef struct block_result block_result_t;
 
 struct my_result {
   fd_pubkey_t block_id;
-  fd_pubkey_t block_hash;
+  fd_pubkey_t bank_hash;
 
   ulong slot;
 
@@ -332,7 +332,7 @@ fd_hard_fork_detector_new( void *        shmem,
   FD_TEST( hf->my_vote_map );
 
   hf->my_vote_lru = my_vote_lru_join( my_vote_lru_new( _my_vote_lru ) );
-  FD_TEST( hf->my_vote_lru ); 
+  FD_TEST( hf->my_vote_lru );
 
   hf->block_id_pool = block_id_pool_join( block_id_pool_new( _block_id_pool, max_live_slots*max_vote_accounts ) );
   FD_TEST( hf->block_id_pool );
@@ -412,16 +412,16 @@ check( fd_hard_fork_detector_t const * detector,
     if( detector->fatal ) FD_LOG_ERR(( "%s", msg ));
     else                  FD_LOG_WARNING(( "%s", msg ));
   } else {
-    if( FD_UNLIKELY( memcmp( my_result->block_hash.uc, block_result->key.block_hash.uc, 32UL ) ) ) {
+    if( FD_UNLIKELY( memcmp( my_result->bank_hash.uc, block_result->key.bank_hash.uc, 32UL ) ) ) {
       char msg[ 4096UL ];
       FD_TEST( fd_cstr_printf_check( msg, sizeof( msg ), NULL,
                                     "HARD FORK DETECTED: our validator has produced block hash `%s` for slot %lu with block ID `%s`, but %lu validators with %.1f of stake have voted on a different block hash `%s` for the same slot",
-                                    FD_BASE58_ENC_32_ALLOCA( my_result->block_hash.uc ),
+                                    FD_BASE58_ENC_32_ALLOCA( my_result->bank_hash.uc ),
                                     my_result->slot,
                                     FD_BASE58_ENC_32_ALLOCA( my_result->block_id.uc ),
                                     block_result->voter_count,
                                     100.0*(double)block_result->stake/(double)detector->total_stake,
-                                    FD_BASE58_ENC_32_ALLOCA( block_result->key.block_hash.uc ) ) );
+                                    FD_BASE58_ENC_32_ALLOCA( block_result->key.bank_hash.uc ) ) );
 
       if( detector->fatal ) FD_LOG_ERR(( "%s", msg ));
       else                  FD_LOG_WARNING(( "%s", msg ));
@@ -433,7 +433,7 @@ void
 fd_hard_fork_detector_vote( fd_hard_fork_detector_t * detector,
                             uchar const *             vote_account,
                             uchar const *             block_id,
-                            uchar const *             block_hash ) {
+                            uchar const *             bank_hash ) {
   stake_t const * _stake = stake_map_ele_query_const( detector->stake_map, fd_type_pun_const( vote_account ), NULL, detector->stake_pool );
   if( FD_UNLIKELY( !_stake ) ) return; /* Don't care about votes from unstaked */
 
@@ -482,9 +482,9 @@ fd_hard_fork_detector_vote( fd_hard_fork_detector_t * detector,
 
   block_result_key_t br_key;
   fd_memcpy( br_key.block_id.uc,   block_id,   32UL );
-  fd_memcpy( br_key.block_hash.uc, block_hash, 32UL );
+  fd_memcpy( br_key.bank_hash.uc, bank_hash, 32UL );
 
-  block_result_t * block_result = block_result_map_ele_query( detector->block_result_map, fd_type_pun_const( block_hash ), NULL, detector->block_result_pool );
+  block_result_t * block_result = block_result_map_ele_query( detector->block_result_map, fd_type_pun_const( bank_hash ), NULL, detector->block_result_pool );
   if( FD_UNLIKELY( !block_result ) ) {
     /* Guaranteed to be space, because the block result pool is larger
        than the voted on pool, and there can be at most one unique
@@ -493,7 +493,7 @@ fd_hard_fork_detector_vote( fd_hard_fork_detector_t * detector,
 
     block_result = block_result_pool_ele_acquire( detector->block_result_pool );
     fd_memcpy( block_result->key.block_id.uc,   block_id,   32UL );
-    fd_memcpy( block_result->key.block_hash.uc, block_hash, 32UL );
+    fd_memcpy( block_result->key.bank_hash.uc, bank_hash, 32UL );
     block_result->stake = 0UL;
     block_result_map_ele_insert( detector->block_result_map, block_result, detector->block_result_pool );
   }
@@ -523,7 +523,7 @@ void
 fd_hard_fork_detector_block( fd_hard_fork_detector_t * detector,
                              ulong                     slot,
                              uchar const *             block_id,
-                             uchar const *             block_hash ) {
+                             uchar const *             bank_hash ) {
   if( FD_LIKELY( !my_vote_pool_free( detector->my_vote_pool ) ) ) {
     /* Might also be good eventually to evict a little bit smarter here ... */
     my_result_t * lru_result = my_vote_lru_ele_pop_tail( detector->my_vote_lru, detector->my_vote_pool );
@@ -534,10 +534,10 @@ fd_hard_fork_detector_block( fd_hard_fork_detector_t * detector,
   my_result_t * my_result = my_vote_pool_ele_acquire( detector->my_vote_pool );
   my_result->slot = slot;
   fd_memcpy( my_result->block_id.uc,   block_id,   32UL );
-  fd_memcpy( my_result->block_hash.uc, block_hash, 32UL );
+  fd_memcpy( my_result->bank_hash.uc, bank_hash, 32UL );
 
   uchar zero32[32] = {0};
-  if( FD_UNLIKELY( !memcmp( block_hash, zero32, 32UL ) ) ) my_result->invalid = 1;
+  if( FD_UNLIKELY( !memcmp( bank_hash, zero32, 32UL ) ) ) my_result->invalid = 1;
 
   my_vote_map_ele_insert( detector->my_vote_map, my_result, detector->my_vote_pool );
   my_vote_lru_ele_push_head( detector->my_vote_lru, my_result, detector->my_vote_pool );

src/discof/tower/fd_hard_fork_detector.h

@@ -27,10 +27,10 @@
    We detect hard forked blocks by monitoring votes from validators, the
    specific criteria is:
 
-    - 52% of stake has voted on a block_id and agreed on a blockhash
-      which is different than the blockhash we have for that block_id.
+    - 52% of stake has voted on a block_id and agreed on a bank_hash
+      which is different than the bank_hash we have for that block_id.
 
-    - 52% of stake has voted on a block_id and agreed on a blockhash
+    - 52% of stake has voted on a block_id and agreed on a bank_hash
       when we marked the block dead because it failed to execute.
 
    Interestingly we do not need to handle the case where we thought a
@@ -42,7 +42,7 @@
 
    We are essentially receiving a
 
-     Stream<Either<(VoteAccountPubkey, BlockId, BlockHash), EpochStakes>>
+     Stream<Either<(VoteAccountPubkey, BlockId, BankHash), EpochStakes>>
 
    The first variant is a vote from some validator for a certain
    block_id and block_hash.  This could come from either any replayed
@@ -51,7 +51,7 @@
    marked dead because it did not validate), all the matters is that the
    vote is validly signed by the vote account.
 
-   Internally, we maintain a Map<BlockId, Map<BlockHash, StakeWeight>>.
+   Internally, we maintain a Map<BlockId, Map<BankHash, StakeWeight>>.
    Any time we receive a vote, if it causes an entry in this map to
    exceed 52% of the total stake weight, or it is from our own
    validator identity, we check if our vote is different, and

src/discof/tower/fd_tower_tile.c

@@ -1,6 +1,7 @@
 #include "fd_tower_tile.h"
 #include "generated/fd_tower_tile_seccomp.h"
 
+#include "fd_hard_fork_detector.h"
 #include "../genesis/fd_genesi_tile.h"
 #include "../../choreo/ghost/fd_ghost.h"
 #include "../../choreo/notar/fd_notar.h"
@@ -93,6 +94,7 @@ typedef struct {
   conf_t *           confs;       /* deque of confirmations queued for publishing */
   fd_banks_t *       banks;
   fd_accdb_user_t    accdb[1];
+  fd_hard_fork_detector_t * hfd;
 
   fd_tower_accts_t  * tower_accts;      /* deque of accts, stake, and pubkey for the currently replayed slot */
   fd_epoch_stakes_t * epoch_stakes;     /* tracks the stakes for each voter in the epoch per fork */
@@ -307,11 +309,14 @@ count_vote_txn( ctx_t *             ctx,
 
   fd_tower_vote_t const * their_last_vote = fd_tower_peek_tail_const( ctx->tower_spare );
   fd_hash_t const *       their_block_id  = &ctx->compact_tower_sync_serde.block_id;
+  fd_hash_t const *       their_bank_hash = &ctx->compact_tower_sync_serde.hash;
 
   ulong            total_stake = fd_ghost_root( ctx->ghost )->total_stake;
   fd_notar_blk_t * notar_blk   = fd_notar_count_vote( ctx->notar, total_stake, vote_acc, their_last_vote->slot, their_block_id );
   if( FD_LIKELY( notar_blk ) ) notar_confirm( ctx, stem, tsorig, notar_blk );
 
+  fd_hard_fork_detector_vote( ctx->hfd, vote_acc->uc, their_block_id->uc, their_bank_hash->uc );
+
   fd_tower_forks_t * fork = fd_forks_query( ctx->forks, their_last_vote->slot );
   if( FD_UNLIKELY( !fork ) ) { ctx->metrics.vote_txn_ignored++; return; /* we haven't replayed this slot yet */ };
 
@@ -455,6 +460,11 @@ replay_slot_completed( ctx_t *                      ctx,
     ctx->conf_slot = slot_info->slot;
   }
 
+  /* Insert the just replayed block into hard fork detector. */
+
+  slot_info->bank_hash.uc[31] &= 0xFE; /* clear lowest bit to ignore minor bank hash differences */
+  fd_hard_fork_detector_block( ctx->hfd, slot_info->slot, slot_info->block_id.uc, slot_info->bank_hash.uc );
+
   /* Insert the just replayed block into ghost. */
 
   fd_hash_t const * parent_block_id = &slot_info->parent_block_id;
@@ -696,7 +706,7 @@ privileged_init( fd_topo_t *      topo,
   ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(ctx_t), sizeof(ctx_t) );
   FD_SCRATCH_ALLOC_FINI( l, scratch_align() );
 
-  FD_TEST( fd_rng_secure( &ctx->seed, 8 ) );
+  FD_TEST( fd_rng_secure( &ctx->seed, sizeof(ctx->seed) ) );
 
   if( FD_UNLIKELY( !strcmp( tile->tower.identity_key, "" ) ) ) FD_LOG_ERR(( "identity_key_path not set" ));
   ctx->identity_key[ 0 ] = *(fd_pubkey_t const *)fd_type_pun_const( fd_keyload_load( tile->tower.identity_key, /* pubkey only: */ 1 ) );

src/flamenco/runtime/program/fd_bpf_loader_program.c

@@ -2624,7 +2624,6 @@ fd_directly_invoke_loader_v3_deploy( fd_bank_t *               bank,
   txn_ctx->xid[0]                    = *xid;
   txn_ctx->progcache                 = NULL;
   txn_ctx->status_cache              = NULL;
-  txn_ctx->bank_hash_cmp             = NULL;
   txn_ctx->log.enable_exec_recording = !!(bank->flags & FD_BANK_FLAGS_EXEC_RECORDING);
   txn_ctx->bank                      = bank;
 

src/flamenco/runtime/tests/fd_instr_harness.c

@@ -94,7 +94,6 @@ fd_solfuzz_pb_instr_ctx_create( fd_solfuzz_runner_t *                runner,
 
   txn_ctx->xid[0]                    = *xid;
   txn_ctx->status_cache              = NULL;
-  txn_ctx->bank_hash_cmp             = NULL;
   txn_ctx->log.enable_exec_recording = !!( runner->bank->flags & FD_BANK_FLAGS_EXEC_RECORDING );
   txn_ctx->bank                      = runner->bank;
 
@@ -352,7 +351,6 @@ fd_solfuzz_pb_instr_ctx_create( fd_solfuzz_runner_t *                runner,
 
   txn_ctx->xid[0]                    = *xid;
   txn_ctx->status_cache              = NULL;
-  txn_ctx->bank_hash_cmp             = NULL;
   txn_ctx->log.enable_exec_recording = !!( runner->bank->flags & FD_BANK_FLAGS_EXEC_RECORDING );
   txn_ctx->bank                      = runner->bank;
 

src/flamenco/runtime/tests/fd_txn_harness.c

@@ -348,7 +348,6 @@ fd_solfuzz_txn_ctx_exec( fd_solfuzz_runner_t *     runner,
   if( FD_UNLIKELY( !txn_ctx->progcache ) ) {
     FD_LOG_CRIT(( "fd_progcache_join failed" ));
   }
-  txn_ctx->bank_hash_cmp = NULL;
   txn_ctx->xid[0]        = *xid;
 
   txn_ctx->log.enable_vm_tracing = runner->enable_vm_tracing;