bank, sched: aggressively mark banks dead and more debug logging

Author: yufeng-jump

src/discof/replay/fd_sched.c

@@ -913,6 +913,7 @@ fd_sched_task_done( fd_sched_t * sched, ulong task_type, ulong txn_idx, ulong ex
       FD_LOG_CRIT(( "invariant violation: active block shouldn't be dying, bank_idx %lu, slot %lu, parent slot %lu",
                     bank_idx, block->slot, block->parent_slot ));
     }
+    FD_LOG_DEBUG(( "dying block %lu drained", block->slot ));
     subtree_abandon( sched, block );
     return;
   }
@@ -948,6 +949,7 @@ fd_sched_block_abandon( fd_sched_t * sched, ulong bank_idx ) {
   subtree_abandon( sched, block );
 
   /* Reset the active block. */
+  FD_LOG_DEBUG(( "reset active_bank_idx %lu", sched->active_bank_idx ));
   sched->active_bank_idx = ULONG_MAX;
   sched->metrics->deactivate_abandoned_cnt++;
   FD_LOG_INFO(( "block %lu abandoned", block->slot ));
@@ -1115,6 +1117,7 @@ fd_sched_root_notify( fd_sched_t * sched, ulong root_idx ) {
         rooted_child_block = child;
       } else {
         /* This is a minority fork. */
+        FD_LOG_DEBUG(( "abandoning minority fork on block %lu", child->slot ));
         subtree_abandon( sched, child );
       }
       child_idx = child->sibling_idx;
@@ -1475,8 +1478,9 @@ subtree_abandon( fd_sched_t * sched, fd_sched_block_t * block ) {
                    !block->staged    || /* parent is in the dispatcher and staged but this block is unstaged */
                    block->staging_lane!=parent->staging_lane; /* this block is on a different staging lane than its parent */
 
-    if( FD_UNLIKELY( in_order && block->staged && sched->active_bank_idx==sched->staged_head_bank_idx[ block->staging_lane ] ) ) {
+    if( FD_UNLIKELY( in_order && block->staged && sched->active_bank_idx==sched->staged_head_bank_idx[ block->staging_lane ] && sched->active_bank_idx!=ULONG_MAX ) ) {
       FD_TEST( block_pool_ele( sched, sched->active_bank_idx )==block );
+      FD_LOG_DEBUG(( "reset active_bank_idx %lu", sched->active_bank_idx ));
       sched->active_bank_idx = ULONG_MAX;
     }
 
@@ -1544,6 +1548,7 @@ maybe_switch_block( fd_sched_t * sched, ulong bank_idx ) {
           if( FD_UNLIKELY( !block_is_activatable( child ) ) ) {
             /* ... but the child is not activatable, likely because
                there are no transactions available yet. */
+            FD_LOG_DEBUG(( "reset active_bank_idx %lu", sched->active_bank_idx ));
             sched->active_bank_idx = ULONG_MAX;
             sched->metrics->deactivate_no_txn_cnt++;
             try_activate_block( sched );
@@ -1562,6 +1567,7 @@ maybe_switch_block( fd_sched_t * sched, ulong bank_idx ) {
         } else {
           /* ... but the child block is considered dead, likely because
              the parser considers it invalid. */
+          FD_LOG_INFO(( "child block %lu is already dead", child->slot ));
           subtree_abandon( sched, child );
           break;
         }
@@ -1574,6 +1580,7 @@ maybe_switch_block( fd_sched_t * sched, ulong bank_idx ) {
     sched->staged_head_bank_idx[ block->staging_lane ] = ULONG_MAX;
 
     /* Reset the active block. */
+    FD_LOG_DEBUG(( "reset active_bank_idx %lu", sched->active_bank_idx ));
     sched->active_bank_idx = ULONG_MAX;
     sched->metrics->deactivate_no_child_cnt++;
     try_activate_block( sched );
@@ -1582,6 +1589,7 @@ maybe_switch_block( fd_sched_t * sched, ulong bank_idx ) {
        are just not getting FEC sets for it fast enough.  This could
        happen when the network path is congested, or when the leader
        simply went down.  Reset the active block. */
+    FD_LOG_DEBUG(( "reset active_bank_idx %lu", sched->active_bank_idx ));
     sched->active_bank_idx = ULONG_MAX;
     sched->metrics->deactivate_no_txn_cnt++;
     try_activate_block( sched );

src/flamenco/runtime/fd_bank.c

@@ -985,14 +985,30 @@ fd_banks_advance_root_prepare( fd_banks_t * banks,
     curr         = fd_banks_pool_ele( bank_pool, curr->parent_idx );
   }
 
-  ulong advance_candidate_idx = prev->idx;
-
   /* If we didn't reach the old root or there is no parent, target is
      not a descendant. */
   if( FD_UNLIKELY( !curr || prev->parent_idx!=root->idx ) ) {
     FD_LOG_CRIT(( "invariant violation: target bank_idx %lu is not a direct descendant of root bank_idx %lu %lu %lu", target_bank_idx, root->idx, prev->idx, prev->parent_idx ));
   }
 
+  curr = root;
+  while( curr && (curr->flags&FD_BANK_FLAGS_ROOTED) && curr!=target_bank ) { /* curr!=target_bank to avoid abandoning good forks. */
+    fd_bank_t * rooted_child = NULL;
+    ulong       child_idx    = curr->child_idx;
+    while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
+      fd_bank_t * child_bank = fd_banks_pool_ele( bank_pool, child_idx );
+      if( child_bank->flags&FD_BANK_FLAGS_ROOTED ) {
+        rooted_child = child_bank;
+      } else {
+        /* This is a minority fork. */
+        FD_LOG_DEBUG(( "abandoning minority fork on bank idx %lu", child_bank->idx ));
+        fd_banks_subtree_mark_dead( bank_pool, child_bank );
+      }
+      child_idx = child_bank->sibling_idx;
+    }
+    curr = rooted_child;
+  }
+
   /* We should mark the old root bank as dead. */
   root->flags |= FD_BANK_FLAGS_DEAD;
 
@@ -1001,11 +1017,11 @@ fd_banks_advance_root_prepare( fd_banks_t * banks,
      potential new root are eligible for pruning.  Each of the sibling
      subtrees can be pruned if the subtrees have no active references on
      their bank. */
+  ulong advance_candidate_idx = prev->idx;
   ulong child_idx = root->child_idx;
   while( child_idx!=fd_banks_pool_idx_null( bank_pool ) ) {
     fd_bank_t * child_bank = fd_banks_pool_ele( bank_pool, child_idx );
     if( child_idx!=advance_candidate_idx ) {
-      fd_banks_subtree_mark_dead( bank_pool, child_bank );
       if( !fd_banks_subtree_can_be_pruned( bank_pool, child_bank ) ) {
         fd_rwlock_unread( &banks->rwlock );
         return 0;