feat(tower): rework confirmed frags and vote txn verify + minor fixes

Author: lidatong

src/app/firedancer/topology.c

@@ -615,15 +615,18 @@ fd_topo_initialize( config_t * config ) {
                        fd_topob_tile_in (   topo, "replay",  0UL,          "metric_in", "snapin_manif", 0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   }
 
-  /**/                 fd_topob_tile_in(    topo, "replay",  0UL,          "metric_in", "poh_replay",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
-  FOR(exec_tile_cnt)   fd_topob_tile_in(    topo, "exec",    i,            "metric_in", "replay_exec",  0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
+  /**/                 fd_topob_tile_in (   topo, "replay",  0UL,          "metric_in", "poh_replay",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
+  FOR(exec_tile_cnt)   fd_topob_tile_in (   topo, "exec",    i,            "metric_in", "replay_exec",  0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
+
+  /**/                 fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "dedup_resolv", 0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   /**/                 fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "genesi_out",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   /**/                 fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "gossip_out",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   /**/                 fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "replay_out",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   if( snapshots_enabled ) {
                        fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "snapin_manif", 0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   }
   /**/                 fd_topob_tile_out(   topo, "tower",   0UL,                       "tower_out",    0UL                                                );
+
   /**/                 fd_topob_tile_in (   topo, "send",    0UL,          "metric_in", "replay_stake", 0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   /**/                 fd_topob_tile_in (   topo, "send",    0UL,          "metric_in", "gossip_out",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   /**/                 fd_topob_tile_in (   topo, "send",    0UL,          "metric_in", "tower_out",    0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );

src/choreo/notar/fd_notar.c

@@ -94,31 +94,23 @@ fd_notar_delete( void * notar ) {
   return notar;
 }
 
-int
+fd_notar_blk_t *
 fd_notar_count_vote( fd_notar_t *        notar,
                      ulong               total_stake,
                      fd_pubkey_t const * addr,
                      ulong               vote_slot,
                      fd_hash_t const *   vote_block_id ) {
 
-  if( FD_UNLIKELY( !notar ) ) {
-    FD_LOG_WARNING(( "NULL notar" ));
-    return 0;
-  }
+  if( FD_UNLIKELY( !notar ) ) { FD_LOG_WARNING(( "NULL notar" )); return NULL; }
 
   /* Ignore if this vote slot isn't in range. */
 
-  if( FD_UNLIKELY( vote_slot < notar->lo_wmark || vote_slot > notar->hi_wmark ) ) return 0;
+  if( FD_UNLIKELY( vote_slot < notar->lo_wmark || vote_slot > notar->hi_wmark ) ) return NULL;
 
   /* Ignore if this vote account isn't in the voter set. */
 
   fd_notar_vtr_t const * vtr = fd_notar_vtr_query( notar->vtr_map, *addr, NULL );
-  if( FD_UNLIKELY( !vtr ) ) return 0;
-
-  /* Ignore if this notar blk was already duplicate confirmed. */
-
-  fd_notar_blk_t * notar_blk = fd_notar_blk_query( notar->blk_map, *vote_block_id, NULL );
-  if( FD_LIKELY( notar_blk && notar_blk->dup_conf ) ) return 0;
+  if( FD_UNLIKELY( !vtr ) ) return NULL;
 
   /* Check we haven't already counted the voter's stake for this slot.
      If a voter voted for multiple block ids for the same slot, we only
@@ -138,12 +130,13 @@ fd_notar_count_vote( fd_notar_t *        notar,
     fd_notar_slot_vtrs_null( notar_slot->prev_vtrs );
     fd_notar_slot_vtrs_null( notar_slot->vtrs );
   }
-  if( FD_LIKELY( fd_notar_slot_vtrs_test( notar_slot->vtrs, vtr->bit ) ) ) return 0;
+  if( FD_LIKELY( fd_notar_slot_vtrs_test( notar_slot->vtrs, vtr->bit ) ) ) return NULL;
   fd_notar_slot_vtrs_insert( notar_slot->vtrs, vtr->bit );
   notar_slot->stake += vtr->stake;
 
   /* Get the actual block with the block_id. */
 
+  fd_notar_blk_t * notar_blk = fd_notar_blk_query( notar->blk_map, *vote_block_id, NULL );
   if( FD_UNLIKELY( !notar_blk ) ) {
     notar_blk        = fd_notar_blk_insert( notar->blk_map, *vote_block_id );
     notar_blk->slot  = vote_slot;
@@ -152,8 +145,9 @@ fd_notar_count_vote( fd_notar_t *        notar,
     notar_slot->block_ids[notar_slot->block_ids_cnt++] = *vote_block_id;
   }
   notar_blk->stake   += vtr->stake;
-  notar_blk->dup_conf = (double)notar_blk->stake / (double)total_stake > 0.52; /* duplicate confirmed if > 52% stake */
-  return notar_blk->dup_conf;
+  notar_blk->dup_conf = ((double)notar_blk->stake / (double)total_stake) > 0.52;
+  notar_blk->opt_conf = ((double)notar_blk->stake / (double)total_stake) > (2.0/3.0);
+  return notar_blk;
 }
 
 void

src/choreo/notar/fd_notar.h

@@ -10,16 +10,16 @@
      important in two contexts:
 
      1. When becoming leader, we need to check that our "previous"
-         leader block _as of_ the parent slot we're building on, has
-         propagated. If it's not propagated, we need to instead
-         retransmit our last block that failed to propagate.  "Previous"
-         is quoted, because there is a grace period of one leader
-         rotation for leader blocks to propagate.
+        leader block _as of_ the parent slot we're building on, has
+        propagated.  If it's not propagated, we need to instead
+        retransmit our last block that failed to propagate.  "Previous"
+        is quoted, because there is a grace period of one leader
+        rotation for leader blocks to propagate.
 
      2. When voting, we need to check our previous leader block _as of_
-         the slot we're voting for has propagated (unless we're voting
-         for one of our leader blocks).  We cannot vote for slots in
-         which our last leader block failed to propagate.
+        the slot we're voting for has propagated (unless we're voting
+        for one of our leader blocks).  We cannot vote for slots in
+        which our last leader block failed to propagate.
 
    - duplicate confirmed: a block is duplicate confirmed if it has
      received votes from at least 52% of stake in the cluster.  The
@@ -85,6 +85,10 @@
 #define FD_NOTAR_PARANOID 1
 #endif
 
+#define FD_NOTAR_FLAG_CONFIRMED_PROPAGATED (0)
+#define FD_NOTAR_FLAG_CONFIRMED_DUPLICATE  (1)
+#define FD_NOTAR_FLAG_CONFIRMED_OPTIMISTIC (2)
+
 #define SET_NAME fd_notar_slot_vtrs
 #define SET_MAX  FD_VOTER_MAX
 #include "../../util/tmpl/fd_set.c"
@@ -95,7 +99,7 @@ struct fd_notar_slot {
   ulong prev_leader_slot; /* previous slot in which we were leader */
   ulong stake;            /* amount of stake that has voted for this slot */
   int   is_leader;        /* whether this slot was our own leader slot */
-  int   is_propagated;    /* whether this slot has reached 33% of stake */
+  int   is_propagated;    /* whether this slot has reached 1/3 of stake */
 
   fd_hash_t block_ids[FD_VOTER_MAX]; /* one block id per voter per slot */
   ulong     block_ids_cnt;           /* count of block ids */
@@ -105,13 +109,13 @@ struct fd_notar_slot {
 };
 typedef struct fd_notar_slot fd_notar_slot_t;
 
-
 struct fd_notar_blk {
   fd_hash_t block_id; /* map key */
   uint      hash;     /* reserved for fd_map_dynamic */
   ulong     slot;     /* slot associated with this block */
   ulong     stake;    /* sum of stake that has voted for this block_id */
   int       dup_conf; /* whether this block has reached 52% of stake */
+  int       opt_conf; /* whether this block has reached 2/3 of stake */
 };
 typedef struct fd_notar_blk fd_notar_blk_t;
 
@@ -161,7 +165,7 @@ struct __attribute__((aligned(128UL))) fd_notar {
   ulong epoch;    /* highest replayed epoch */
   ulong lo_wmark; /* notar ignores votes < lo_wmark */
   ulong hi_wmark; /* notar ignores votes > hi_wmark */
-  ulong slot_max; /* maximum slot number */
+  ulong slot_max; /* maximum number of slots notar can track */
 
   fd_notar_slot_t * slot_map; /* tracks who has voted for a given slot */
   fd_notar_blk_t *  blk_map;  /* tracks amount of stake for a given block (keyed by block id) */
@@ -237,7 +241,7 @@ fd_notar_delete( void * notar );
    account in the current epoch, slot is slot being voted for, block_id
    is the voter's proposed block id for this vote slot. */
 
-int
+fd_notar_blk_t *
 fd_notar_count_vote( fd_notar_t *        notar,
                      ulong               total_stake,
                      fd_pubkey_t const * addr,

src/choreo/tower/fd_tower.c

@@ -641,7 +641,7 @@ fd_tower_to_vote_txn( fd_tower_t const *    tower,
   for( fd_tower_iter_t iter = fd_tower_iter_init( tower       );
                              !fd_tower_iter_done( tower, iter );
                        iter = fd_tower_iter_next( tower, iter ) ) {
-    fd_tower_t const * vote              = fd_tower_iter_ele_const( tower, iter );
+    fd_tower_t const * vote                   = fd_tower_iter_ele_const( tower, iter );
     tower_sync.lockouts[i].offset             = vote->slot - prev;
     tower_sync.lockouts[i].confirmation_count = (uchar)vote->conf;
     prev                                      = vote->slot;

src/disco/gui/fd_gui.c

@@ -2428,27 +2428,22 @@ fd_gui_handle_rooted_slot( fd_gui_t * gui, ulong root_slot ) {
   fd_http_server_ws_broadcast( gui->http );
 }
 
-/* fd_gui_handle_tower_update handles updates from the tower tile, which
-   manages consensus related fork switching, rooting, slot confirmation. */
+/* fd_gui_handle_tower_slot_done handles slot_done frags from the tower
+   tile, which provides the reset slot and vote slot. */
 void
-fd_gui_handle_tower_update( fd_gui_t *                   gui,
-                            fd_tower_slot_done_t const * tower,
-                            long                         now ) {
+fd_gui_handle_tower_slot_done( fd_gui_t *                   gui,
+                               fd_tower_slot_done_t const * slot_done,
+                               long                         now ) {
   (void)now;
 
-  /* handle new root */
-  if( FD_LIKELY( tower->root_slot!=ULONG_MAX && gui->summary.slot_rooted!=tower->root_slot ) ) {
-    fd_gui_handle_rooted_slot( gui, tower->root_slot );
-  }
-
-  if( FD_UNLIKELY( gui->summary.vote_distance!=tower->reset_slot-tower->vote_slot ) ) {
-    gui->summary.vote_distance = tower->reset_slot-tower->vote_slot;
+  if( FD_UNLIKELY( gui->summary.vote_distance!=slot_done->reset_slot-slot_done->vote_slot ) ) {
+    gui->summary.vote_distance = slot_done->reset_slot-slot_done->vote_slot; /* TODO doesn't seem correct? */
     fd_gui_printf_vote_distance( gui );
     fd_http_server_ws_broadcast( gui->http );
   }
 
   if( FD_LIKELY( gui->summary.vote_state!=FD_GUI_VOTE_STATE_NON_VOTING ) ) {
-    if( FD_UNLIKELY( tower->vote_slot==ULONG_MAX || (tower->vote_slot+150UL)<tower->reset_slot ) ) {
+    if( FD_UNLIKELY( slot_done->vote_slot==ULONG_MAX || (slot_done->vote_slot+150UL)<slot_done->reset_slot ) ) {
       if( FD_UNLIKELY( gui->summary.vote_state!=FD_GUI_VOTE_STATE_DELINQUENT ) ) {
         gui->summary.vote_state = FD_GUI_VOTE_STATE_DELINQUENT;
         fd_gui_printf_vote_state( gui );
@@ -2462,6 +2457,22 @@ fd_gui_handle_tower_update( fd_gui_t *                   gui,
       }
     }
   }
+}
+
+/* fd_gui_handle_tower_slot_confirmed handles slot_confirmed frags from
+   the tower tile, which indicate when slots are optimistically
+   confirmed or rooted. */
+
+void
+fd_gui_handle_tower_slot_confirmed( fd_gui_t *                        gui,
+                                    fd_tower_slot_confirmed_t const * slot_confirmed,
+                                    long                              now ) {
+  (void)now;
+
+  /* handle new root */
+  if( FD_LIKELY( slot_confirmed->kind==FD_TOWER_SLOT_CONFIRMED_ROOTED && gui->summary.slot_rooted!=slot_confirmed->slot ) ) {
+    fd_gui_handle_rooted_slot( gui, slot_confirmed->slot );
+  }
 
   /* todo ... optimistic confirmation, waiting on fd_ghost / fd_notar */
 }

src/disco/gui/fd_gui.h

@@ -833,10 +833,15 @@ fd_gui_handle_leader_schedule( fd_gui_t *                    gui,
                                long                          now );
 
 void
-fd_gui_handle_tower_update( fd_gui_t *                   gui,
+fd_gui_handle_tower_slot_done( fd_gui_t *                   gui,
                             fd_tower_slot_done_t const * msg,
                             long                         now );
 
+void
+fd_gui_handle_tower_slot_confirmed( fd_gui_t *                        gui,
+                                    fd_tower_slot_confirmed_t const * slot_confirmed,
+                                    long                              now );
+
 void
 fd_gui_handle_replay_update( fd_gui_t *                gui,
                              fd_gui_slot_completed_t * slot_completed,

src/disco/gui/fd_gui_tile.c

@@ -385,8 +385,11 @@ after_frag( fd_gui_ctx_t *      ctx,
     case IN_KIND_TOWER_OUT: {
       FD_TEST( ctx->is_full_client );
       if( FD_LIKELY( sig==FD_TOWER_SIG_SLOT_DONE )) {
-        fd_tower_slot_done_t const * tower = (fd_tower_slot_done_t const *)ctx->buf;
-        fd_gui_handle_tower_update( ctx->gui, tower, fd_clock_now( ctx->clock ) );
+        fd_tower_slot_done_t const * slot_done = (fd_tower_slot_done_t const *)ctx->buf;
+        fd_gui_handle_tower_slot_done( ctx->gui, slot_done, fd_clock_now( ctx->clock ) );
+      } else if ( FD_LIKELY( sig==FD_TOWER_SIG_SLOT_CONFIRMED ) ) {
+        fd_tower_slot_confirmed_t const * slot_confirmed = (fd_tower_slot_confirmed_t const *)ctx->buf;
+        fd_gui_handle_tower_slot_confirmed( ctx->gui, slot_confirmed, fd_clock_now( ctx->clock ) );
       }
       break;
     }

src/discof/backtest/fd_backtest_tile.c

@@ -308,8 +308,6 @@ returnable_frag( fd_backt_tile_t *   ctx,
       dst->vote_slot             = msg->slot;
       dst->reset_slot            = msg->slot;
       dst->reset_block_id        = msg->block_id;
-      dst->root_slot             = msg->slot;
-      dst->root_block_id         = msg->block_id;
 
       fd_stem_publish( stem, ctx->tower_out->idx, 0UL, ctx->tower_out->chunk, sizeof(fd_tower_slot_done_t), 0UL, tspub, fd_frag_meta_ts_comp( fd_tickcount() ) );
       ctx->tower_out->chunk = fd_dcache_compact_next( ctx->tower_out->chunk, sizeof(fd_tower_slot_done_t), ctx->tower_out->chunk0, ctx->tower_out->wmark );

src/discof/repair/fd_repair_tile.c

@@ -795,8 +795,12 @@ after_frag( ctx_t * ctx,
   }
 
   if( FD_UNLIKELY( in_kind==IN_KIND_TOWER ) ) {
-    fd_tower_slot_done_t const * msg = (fd_tower_slot_done_t const *)fd_type_pun_const( ctx->buffer );
-    if( FD_LIKELY( msg->new_root ) ) fd_forest_publish( ctx->forest, msg->root_slot );
+    if( FD_LIKELY( sig == FD_TOWER_SIG_SLOT_CONFIRMED ) ) {
+      fd_tower_slot_confirmed_t const * msg = (fd_tower_slot_confirmed_t const *)fd_type_pun_const( ctx->buffer );
+      if( FD_LIKELY( msg->kind==FD_TOWER_SLOT_CONFIRMED_ROOTED ) ) {
+        fd_forest_publish( ctx->forest, msg->slot   );
+      }
+    }
     return;
   }
 
@@ -806,12 +810,14 @@ after_frag( ctx_t * ctx,
   }
 
   if( FD_UNLIKELY( in_kind==IN_KIND_SHRED ) ) {
+
     /* There are 3 message types from shred:
         1. resolver evict - incomplete FEC set is evicted by resolver
         2. fec complete   - FEC set is completed by resolver. Also contains a shred.
         3. shred          - new shred
 
         Msgs 2 and 3 have a shred header in ctx->buffer */
+
     int resolver_evicted = sz == 0;
     int fec_completes    = sz == FD_SHRED_DATA_HEADER_SZ + sizeof(fd_hash_t) + sizeof(fd_hash_t) + sizeof(int);
     if( FD_UNLIKELY( resolver_evicted ) ) {

src/discof/replay/fd_replay_tile.c

@@ -146,6 +146,7 @@ struct fd_replay_tile {
   fd_txncache_t * txncache;
   fd_store_t *    store;
   fd_banks_t *    banks;
+  ulong           reset_bank_idx; /* most recent reset bank idx, used for determining root distance */
 
   /* This flag is 1 If we have seen a vote signature that our node has
      sent out get rooted at least one time.  The value is 0 otherwise.
@@ -2123,9 +2124,9 @@ process_exec_task_done( fd_replay_tile_t *        ctx,
 }
 
 static void
-process_tower_update( fd_replay_tile_t *           ctx,
-                      fd_stem_context_t *          stem,
-                      fd_tower_slot_done_t const * msg ) {
+process_tower_slot_done( fd_replay_tile_t *           ctx,
+                         fd_stem_context_t *          stem,
+                         fd_tower_slot_done_t const * msg ) {
   ctx->reset_block_id = msg->reset_block_id;
   ctx->reset_slot     = msg->reset_slot;
   ctx->reset_timestamp_nanos = fd_log_wallclock();
@@ -2148,7 +2149,6 @@ process_tower_update( fd_replay_tile_t *           ctx,
     FD_LOG_CRIT(( "invariant violation: bank not found for bank index %lu", reset_bank_idx ));
   }
 
-  if( FD_LIKELY( msg->root_slot!=ULONG_MAX ) ) FD_TEST( msg->root_slot<=msg->reset_slot );
   ctx->reset_bank = bank;
 
   if( FD_LIKELY( ctx->replay_out->idx!=ULONG_MAX ) ) {
@@ -2184,22 +2184,36 @@ process_tower_update( fd_replay_tile_t *           ctx,
     ctx->replay_out->chunk = fd_dcache_compact_next( ctx->replay_out->chunk, sizeof(fd_poh_reset_t), ctx->replay_out->chunk0, ctx->replay_out->wmark );
   }
 
-  FD_LOG_INFO(( "tower_update(reset_slot=%lu, next_leader_slot=%lu, vote_slot=%lu, root_slot=%lu, root_block_id=%s)", msg->reset_slot, ctx->next_leader_slot, msg->vote_slot, msg->root_slot, FD_BASE58_ENC_32_ALLOCA( &msg->root_block_id ) ));
+  FD_LOG_INFO(( "tower_slot_done(reset_slot=%lu, next_leader_slot=%lu, vote_slot=%lu)", msg->reset_slot, ctx->next_leader_slot, msg->vote_slot ));
   maybe_become_leader( ctx, stem );
+}
 
-  if( FD_LIKELY( msg->root_slot!=ULONG_MAX ) ) {
+static void
+process_tower_slot_confirmed( fd_replay_tile_t *                ctx,
+                              fd_stem_context_t *               stem,
+                              fd_tower_slot_confirmed_t const * msg ) {
+  if( FD_LIKELY( msg->kind==FD_TOWER_SLOT_CONFIRMED_ROOTED ) ) {
 
-    FD_TEST( msg->root_slot>=ctx->consensus_root_slot );
-    fd_block_id_ele_t * block_id_ele = fd_block_id_map_ele_query( ctx->block_id_map, &msg->root_block_id, NULL, ctx->block_id_arr );
+    FD_TEST( msg->slot>=ctx->consensus_root_slot );
+    fd_block_id_ele_t * block_id_ele = fd_block_id_map_ele_query( ctx->block_id_map, &msg->block_id, NULL, ctx->block_id_arr );
     FD_TEST( block_id_ele );
 
-    ctx->consensus_root_slot     = msg->root_slot;
-    ctx->consensus_root          = msg->root_block_id;
+    ctx->consensus_root_slot     = msg->slot;
+    ctx->consensus_root          = msg->block_id;
     ctx->consensus_root_bank_idx = fd_block_id_ele_get_idx( ctx->block_id_arr, block_id_ele );
 
     publish_root_advanced( ctx, stem );
   }
 
+  /* If tower roots a slot, it always publishes the frag immediately
+     after the slot_done message, so ctx->reset_bank_idx should still be
+     valid at this point (it could not have been published away).  */
+
+  fd_bank_t * bank = fd_banks_bank_query( ctx->banks, ctx->reset_bank_idx );
+  if( FD_UNLIKELY( !bank ) ) {
+    FD_LOG_CRIT(( "invariant violation: bank not found for bank index %lu", ctx->reset_bank_idx ));
+  }
+
   ulong distance = 0UL;
   fd_bank_t * parent = bank;
   while( parent ) {
@@ -2369,7 +2383,8 @@ returnable_frag( fd_replay_tile_t *  ctx,
       break;
     }
     case IN_KIND_TOWER: {
-      if( FD_LIKELY( sig==FD_TOWER_SIG_SLOT_DONE ) ) process_tower_update( ctx, stem, fd_chunk_to_laddr( ctx->in[ in_idx ].mem, chunk ) );
+      if     ( FD_LIKELY( sig==FD_TOWER_SIG_SLOT_DONE      ) ) process_tower_slot_done     ( ctx, stem, fd_chunk_to_laddr( ctx->in[ in_idx ].mem, chunk ) );
+      else if( FD_LIKELY( sig==FD_TOWER_SIG_SLOT_CONFIRMED ) ) process_tower_slot_confirmed( ctx, stem, fd_chunk_to_laddr( ctx->in[ in_idx ].mem, chunk ) );
       break;
     }
     case IN_KIND_SHRED: {