tower: add hard fork detector

book/api/metrics-generated.md

@@ -1195,5 +1195,9 @@
 | <span class="metrics-name">tower_&#8203;lockout_&#8203;fail</span> | counter | Locked out (can't vote) |
 | <span class="metrics-name">tower_&#8203;threshold_&#8203;fail</span> | counter | Did not pass threshold check (can't vote) |
 | <span class="metrics-name">tower_&#8203;propagated_&#8203;fail</span> | counter | Prev leader block did not propagate (can't vote) |
+| <span class="metrics-name">tower_&#8203;hard_&#8203;forks_&#8203;seen</span> | counter | Number of hard forks we've seen (block ids with multiple candidate bank hashes) |
+| <span class="metrics-name">tower_&#8203;hard_&#8203;forks_&#8203;pruned</span> | counter | Number of hard forks (candidate bank hashes) we've pruned |
+| <span class="metrics-name">tower_&#8203;hard_&#8203;forks_&#8203;active</span> | gauge | Currently active hard forks |
+| <span class="metrics-name">tower_&#8203;hard_&#8203;forks_&#8203;max_&#8203;width</span> | gauge | The max width of hard forks (block id with most candidate bank hashes) we've ever seen |
 
 </div>

src/app/firedancer-dev/commands/backtest.c

@@ -313,13 +313,6 @@ backtest_topo( config_t * config ) {
   FOR(exec_tile_cnt) fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "exec", i ) ], banks_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
   FD_TEST( fd_pod_insertf_ulong( topo->props, banks_obj->id, "banks" ) );
 
-  /* bank_hash_cmp_obj shared by replay and exec tiles */
-  fd_topob_wksp( topo, "bh_cmp" );
-  fd_topo_obj_t * bank_hash_cmp_obj = setup_topo_bank_hash_cmp( topo, "bh_cmp" );
-  fd_topob_tile_uses( topo, replay_tile, bank_hash_cmp_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
-  FOR(exec_tile_cnt) fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "exec", i ) ], bank_hash_cmp_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
-  FD_TEST( fd_pod_insertf_ulong( topo->props, bank_hash_cmp_obj->id, "bh_cmp" ) );
-
   /* txncache_obj, busy_obj and poh_slot_obj only by replay tile */
   fd_topob_wksp( topo, "txncache"    );
   fd_topob_wksp( topo, "bank_busy"   );
@@ -358,7 +351,6 @@ backtest_topo( config_t * config ) {
     fd_topo_configure_tile( tile, config );
 
     if( !strcmp( tile->name, "replay" ) ) {
-      tile->replay.enable_bank_hash_cmp = 0;
       tile->replay.enable_features_cnt = config->tiles.replay.enable_features_cnt;
       for( ulong i = 0; i < tile->replay.enable_features_cnt; i++ ) {
         strncpy( tile->replay.enable_features[i], config->tiles.replay.enable_features[i], sizeof(tile->replay.enable_features[i]) );

src/app/firedancer-dev/main.c

@@ -24,7 +24,6 @@ extern fd_topo_obj_callbacks_t fd_obj_cb_fec_sets;
 extern fd_topo_obj_callbacks_t fd_obj_cb_txncache;
 extern fd_topo_obj_callbacks_t fd_obj_cb_banks;
 extern fd_topo_obj_callbacks_t fd_obj_cb_funk;
-extern fd_topo_obj_callbacks_t fd_obj_cb_bank_hash_cmp;
 
 extern fd_topo_obj_callbacks_t fd_obj_cb_vinyl_meta;
 extern fd_topo_obj_callbacks_t fd_obj_cb_vinyl_meta_ele;
@@ -47,7 +46,6 @@ fd_topo_obj_callbacks_t * CALLBACKS[] = {
   &fd_obj_cb_txncache,
   &fd_obj_cb_banks,
   &fd_obj_cb_funk,
-  &fd_obj_cb_bank_hash_cmp,
   &fd_obj_cb_vinyl_meta,
   &fd_obj_cb_vinyl_meta_ele,
   &fd_obj_cb_vinyl_data,

src/app/firedancer/callbacks.c

@@ -38,34 +38,6 @@ fd_topo_obj_callbacks_t fd_obj_cb_banks = {
   .new       = banks_new,
 };
 
-static ulong
-bh_cmp_footprint( fd_topo_t const *     topo,
-                  fd_topo_obj_t const * obj ) {
-  (void)topo; (void)obj;
-  return fd_bank_hash_cmp_footprint();
-}
-
-static ulong
-bh_cmp_align( fd_topo_t const *     topo,
-              fd_topo_obj_t const * obj ) {
-  (void)topo; (void)obj;
-  return fd_bank_hash_cmp_align();
-}
-
-static void
-bh_cmp_new( fd_topo_t const *     topo,
-                   fd_topo_obj_t const * obj ) {
-  (void)topo; (void)obj;
-  FD_TEST( fd_bank_hash_cmp_new( fd_topo_obj_laddr( topo, obj->id ) ) );
-}
-
-fd_topo_obj_callbacks_t fd_obj_cb_bank_hash_cmp = {
-  .name      = "bh_cmp",
-  .footprint = bh_cmp_footprint,
-  .align     = bh_cmp_align,
-  .new       = bh_cmp_new,
-};
-
 static ulong
 funk_align( fd_topo_t const *     topo,
             fd_topo_obj_t const * obj ) {

src/app/firedancer/config/default.toml

@@ -1371,15 +1371,15 @@ user = ""
     [tiles.tower]
         # Solana reaches consensus via replay, but can "cluster confirm"
         # slots ahead of the replay tip by listening to vote txns from
-        # gossip or TPU.  The larger max_lookahead_conf, the further
+        # gossip or TPU.  The larger max_vote_lookahead, the further
         # ahead slots can be cluster confirmed before they are replayed.
         #
         # Specifically, tower will ignore gossip or TPU votes that are
-        # more than max_lookahead_conf slots ahead of the root.
+        # more than max_vote_lookahead slots ahead of the root.
         #
-        # Note max_lookahead_conf must be >= max_live_slots and
+        # Note max_vote_lookahead must be >= max_live_slots and
         # Firedancer will ignore a value where this is not the case.
-        max_lookahead_conf = 4096
+        max_vote_lookahead = 4096
 
     [tiles.send]
         # The port the send tile uses for QUIC, to send votes and other
@@ -1528,6 +1528,19 @@ user = ""
     # enabled during routine running of the validator.
     core_dump = true
 
+    # A hard fork occurs when our validator has some implementation
+    # difference in consensus rules and gets a different blockhash for
+    # a given slot than the rest of the cluster.
+    #
+    # Typically, we should assume that our validator is correct and keep
+    # running, but during development it can be useful to detect if we
+    # are on the minority hard fork and immediately exit or abort.
+    #
+    # If this option is true, the validator will immediately exit with
+    # an error if it detects a hard fork.  If false, the validator will
+    # log the hard fork event and continue running.
+    hard_fork_fatal = true
+
     # It can be convenient during development to use a network namespace
     # for running Firedancer.  This allows us to send packets at a local
     # Firedancer instance and have them go through more of the kernel

src/app/firedancer/main.c

@@ -21,7 +21,6 @@ extern fd_topo_obj_callbacks_t fd_obj_cb_fec_sets;
 extern fd_topo_obj_callbacks_t fd_obj_cb_txncache;
 extern fd_topo_obj_callbacks_t fd_obj_cb_banks;
 extern fd_topo_obj_callbacks_t fd_obj_cb_funk;
-extern fd_topo_obj_callbacks_t fd_obj_cb_bank_hash_cmp;
 
 fd_topo_obj_callbacks_t * CALLBACKS[] = {
   &fd_obj_cb_mcache,
@@ -39,7 +38,6 @@ fd_topo_obj_callbacks_t * CALLBACKS[] = {
   &fd_obj_cb_txncache,
   &fd_obj_cb_banks,
   &fd_obj_cb_funk,
-  &fd_obj_cb_bank_hash_cmp,
   NULL,
 };
 

src/app/firedancer/topology.c

@@ -52,12 +52,6 @@ parse_ip_port( const char * name, const char * ip_port, fd_topo_ip_port_t *parse
     FD_LOG_ERR(( "could not parse port %s in [%s]", ip_end+1, name ));
 }
 
-fd_topo_obj_t *
-setup_topo_bank_hash_cmp( fd_topo_t * topo, char const * wksp_name ) {
-  fd_topo_obj_t * obj = fd_topob_obj( topo, "bh_cmp", wksp_name );
-  return obj;
-}
-
 fd_topo_obj_t *
 setup_topo_banks( fd_topo_t *  topo,
                   char const * wksp_name,
@@ -351,7 +345,6 @@ fd_topo_initialize( config_t * config ) {
 
   fd_topob_wksp( topo, "funk"         );
   fd_topob_wksp( topo, "progcache"    );
-  fd_topob_wksp( topo, "bh_cmp"       );
   fd_topob_wksp( topo, "fec_sets"     );
   fd_topob_wksp( topo, "txncache"     );
   fd_topob_wksp( topo, "banks"        );
@@ -940,12 +933,6 @@ fd_topo_initialize( config_t * config ) {
     }
   }
 
-  /* TODO: This should not exist in production */
-  fd_topo_obj_t * bank_hash_cmp_obj = setup_topo_bank_hash_cmp( topo, "bh_cmp" );
-  /**/               fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "replay", 0UL ) ], bank_hash_cmp_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
-  FOR(exec_tile_cnt) fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "exec",   i   ) ], bank_hash_cmp_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
-  FD_TEST( fd_pod_insertf_ulong( topo->props, bank_hash_cmp_obj->id, "bh_cmp" ) );
-
   ulong fec_set_cnt = shred_depth + config->tiles.shred.max_pending_shred_sets + 4UL;
   ulong fec_sets_sz = fec_set_cnt*sizeof(fd_shred34_t)*4; /* mirrors # of dcache entires in frankendancer */
   fd_topo_obj_t * fec_sets_obj = setup_topo_fec_sets( topo, "fec_sets", shred_tile_cnt*fec_sets_sz );
@@ -1185,7 +1172,6 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
     strncpy( tile->replay.identity_key_path, config->paths.identity_key, sizeof(tile->replay.identity_key_path) );
     tile->replay.ip_addr = config->net.ip_addr;
     strncpy( tile->replay.vote_account_path, config->paths.vote_account, sizeof(tile->replay.vote_account_path) );
-    tile->replay.enable_bank_hash_cmp = 1;
 
     tile->replay.capture_start_slot = config->capture.capture_start_slot;
     strncpy( tile->replay.solcap_capture, config->capture.solcap_capture, sizeof(tile->replay.solcap_capture) );
@@ -1212,8 +1198,9 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
 
   } else if( FD_UNLIKELY( !strcmp( tile->name, "tower" ) ) ) {
 
+    tile->tower.hard_fork_fatal    = config->firedancer.development.hard_fork_fatal;
     tile->tower.max_live_slots     = config->firedancer.runtime.max_live_slots;
-    tile->tower.max_lookahead_conf = config->tiles.tower.max_lookahead_conf;
+    tile->tower.max_vote_lookahead = config->tiles.tower.max_vote_lookahead;
     strncpy( tile->tower.identity_key, config->paths.identity_key, sizeof(tile->tower.identity_key) );
     strncpy( tile->tower.vote_account, config->paths.vote_account, sizeof(tile->tower.vote_account) );
     strncpy( tile->tower.base_path, config->paths.base, sizeof(tile->tower.base_path) );

src/app/firedancer/topology.h

@@ -13,9 +13,6 @@ FD_PROTOTYPES_BEGIN
 void
 fd_topo_initialize( fd_config_t * config );
 
-fd_topo_obj_t *
-setup_topo_bank_hash_cmp( fd_topo_t * topo, char const * wksp_name );
-
 fd_topo_obj_t *
 setup_topo_banks( fd_topo_t *  topo,
                   char const * wksp_name,

src/app/shared/fd_config.h

@@ -161,6 +161,10 @@ struct fd_configf {
     uint full_effective_age_cancel_threshold;
   } snapshots;
 
+  struct {
+    int hard_fork_fatal;
+  } development;
+
   struct {
     ulong max_completed_shred_sets;
   } store;
@@ -486,7 +490,7 @@ struct fd_config {
     } shredcap;
 
     struct {
-      ulong max_lookahead_conf;
+      ulong max_vote_lookahead;
     } tower;
 
   } tiles;

src/app/shared/fd_config_parse.c

@@ -118,6 +118,8 @@ fd_config_extract_podf( uchar *        pod,
   CFG_POP      ( uint,   snapshots.max_incremental_snapshots_to_keep         );
   CFG_POP      ( uint,   snapshots.full_effective_age_cancel_threshold       );
 
+  CFG_POP      ( bool,   development.hard_fork_fatal                         );
+
   return config;
 }
 
@@ -257,7 +259,7 @@ fd_config_extract_pod( uchar *       pod,
 
   CFG_POP      ( ushort, tiles.send.send_src_port                         );
 
-  CFG_POP      ( ulong,  tiles.tower.max_lookahead_conf                   );
+  CFG_POP      ( ulong,  tiles.tower.max_vote_lookahead                   );
 
   CFG_POP      ( bool,   tiles.archiver.enabled                           );
   CFG_POP      ( ulong,  tiles.archiver.end_slot                          );

src/app/shared/fd_tile_unit_test.c

@@ -21,7 +21,6 @@ extern fd_topo_obj_callbacks_t fd_obj_cb_fec_sets;
 extern fd_topo_obj_callbacks_t fd_obj_cb_txncache;
 extern fd_topo_obj_callbacks_t fd_obj_cb_banks;
 extern fd_topo_obj_callbacks_t fd_obj_cb_funk;
-extern fd_topo_obj_callbacks_t fd_obj_cb_bank_hash_cmp;
 
 fd_topo_obj_callbacks_t * CALLBACKS[] = {
   &fd_obj_cb_mcache,
@@ -39,7 +38,6 @@ fd_topo_obj_callbacks_t * CALLBACKS[] = {
   &fd_obj_cb_txncache,
   &fd_obj_cb_banks,
   &fd_obj_cb_funk,
-  &fd_obj_cb_bank_hash_cmp,
   NULL,
 };
 

src/choreo/fd_choreo_base.h

@@ -36,5 +36,6 @@ typedef uchar fd_block_id_t[ 32UL ];
 typedef fd_slot_hash_t fd_slot_pubkey_t;
 
 static const fd_pubkey_t pubkey_null = {{ 0 }};
+static const fd_hash_t hash_null = {{ 0 }};
 
 #endif /* HEADER_fd_src_choreo_fd_choreo_base_h */

src/choreo/ghost/fd_ghost.c

@@ -271,18 +271,18 @@ fd_ghost_insert( fd_ghost_t      * ghost,
 void
 fd_ghost_count_vote( fd_ghost_t *        ghost,
                      fd_ghost_blk_t *    blk,
-                     fd_pubkey_t const * vtr_addr,
+                     fd_pubkey_t const * vote_acc,
                      ulong               stake,
                      ulong               slot ) {
 
   fd_ghost_blk_t const * root = fd_ghost_root( ghost );
   fd_ghost_blk_t *       pool = ghost->pool;
-  fd_ghost_vtr_t *       vtr  = vtr_map_query( ghost->vtr_map, *vtr_addr, NULL );
+  fd_ghost_vtr_t *       vtr  = vtr_map_query( ghost->vtr_map, *vote_acc, NULL );
 
   if( FD_UNLIKELY( slot == ULONG_MAX  ) ) return; /* hasn't voted */
   if( FD_UNLIKELY( slot <  root->slot ) ) return; /* vote older than root */
 
-  if( FD_UNLIKELY( !vtr ) ) vtr = vtr_map_insert( ghost->vtr_map, *vtr_addr );
+  if( FD_UNLIKELY( !vtr ) ) vtr = vtr_map_insert( ghost->vtr_map, *vote_acc );
   else {
 
     /* Only process the vote if it is not the same as the previous vote

src/choreo/hfork/Local.mk

@@ -0,0 +1,8 @@
+$(call add-hdrs,fd_hfork.h)
+$(call add-objs,fd_hfork,fd_choreo)
+ifdef FD_HAS_HOSTED
+ifdef FD_HAS_SECP256K1
+$(call make-unit-test,test_hfork,test_hfork,fd_choreo fd_flamenco fd_tango fd_ballet fd_util)
+$(call run-unit-test,test_hfork)
+endif
+endif