resolf: fix accdb races

The full Firedancer resolv tile was not synchronized with database
admin operations performed by the replay tile.  This resulted in
assertion failures and crashes.

This patch makes the resolv tile use a speculative query API
(accdb_peek) and gracefully skip over such data races.

Author: riptl

book/api/metrics-generated.md

@@ -1054,6 +1054,7 @@
 | <span class="metrics-name">resolf_&#8203;lut_&#8203;resolved</span><br/>{lut_&#8203;resolve_&#8203;result="<span class="metrics-enum">success</span>"} | counter | Count of address lookup tables resolved (Resolved successfully) |
 | <span class="metrics-name">resolf_&#8203;blockhash_&#8203;expired</span> | counter | Count of transactions that failed to resolve because the blockhash was expired |
 | <span class="metrics-name">resolf_&#8203;transaction_&#8203;bundle_&#8203;peer_&#8203;failure</span> | counter | Count of transactions that failed to resolve because a peer transaction in the bundle failed |
+| <span class="metrics-name">resolf_&#8203;db_&#8203;races</span> | counter | Number of database races encountered (diagnostic counter, not indicative of issues) |
 
 </div>
 

src/disco/metrics/generated/fd_metrics_resolf.c

@@ -15,4 +15,5 @@ const fd_metrics_meta_t FD_METRICS_RESOLF[FD_METRICS_RESOLF_TOTAL] = {
     DECLARE_METRIC_ENUM( RESOLF_LUT_RESOLVED, COUNTER, LUT_RESOLVE_RESULT, SUCCESS ),
     DECLARE_METRIC( RESOLF_BLOCKHASH_EXPIRED, COUNTER ),
     DECLARE_METRIC( RESOLF_TRANSACTION_BUNDLE_PEER_FAILURE, COUNTER ),
+    DECLARE_METRIC( RESOLF_DB_RACES, COUNTER ),
 };

src/disco/metrics/generated/fd_metrics_resolf.h

@@ -50,7 +50,13 @@
 #define FD_METRICS_COUNTER_RESOLF_TRANSACTION_BUNDLE_PEER_FAILURE_DESC "Count of transactions that failed to resolve because a peer transaction in the bundle failed"
 #define FD_METRICS_COUNTER_RESOLF_TRANSACTION_BUNDLE_PEER_FAILURE_CVT  (FD_METRICS_CONVERTER_NONE)
 
-#define FD_METRICS_RESOLF_TOTAL (13UL)
+#define FD_METRICS_COUNTER_RESOLF_DB_RACES_OFF  (29UL)
+#define FD_METRICS_COUNTER_RESOLF_DB_RACES_NAME "resolf_db_races"
+#define FD_METRICS_COUNTER_RESOLF_DB_RACES_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_RESOLF_DB_RACES_DESC "Number of database races encountered (diagnostic counter, not indicative of issues)"
+#define FD_METRICS_COUNTER_RESOLF_DB_RACES_CVT  (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_RESOLF_TOTAL (14UL)
 extern const fd_metrics_meta_t FD_METRICS_RESOLF[FD_METRICS_RESOLF_TOTAL];
 
 #endif /* HEADER_fd_src_disco_metrics_generated_fd_metrics_resolf_h */

src/disco/metrics/metrics.xml

@@ -432,6 +432,7 @@ metric introduced.
     <counter name="LutResolved" enum="LutResolveResult" summary="Count of address lookup tables resolved" />
     <counter name="BlockhashExpired" summary="Count of transactions that failed to resolve because the blockhash was expired" />
     <counter name="TransactionBundlePeerFailure" summary="Count of transactions that failed to resolve because a peer transaction in the bundle failed" />
+    <counter name="DbRaces" summary="Number of database races encountered (diagnostic counter, not indicative of issues)" />
 </tile>
 
 <tile name="resolv">

src/discof/bank/fd_bank_err.h

@@ -59,24 +59,4 @@ fd_bank_err_from_runtime_err( int err ) {
     return 0;
 }
 
-#define FD_BANK_LUT_ERR_SUCCESS                            ( 0)
-#define FD_BANK_LUT_ERR_ADDRESS_LOOKUP_TABLE_NOT_FOUND     (-1)
-#define FD_BANK_LUT_ERR_INVALID_ADDRESS_LOOKUP_TABLE_OWNER (-2)
-#define FD_BANK_LUT_ERR_INVALID_ADDRESS_LOOKUP_TABLE_DATA  (-3)
-#define FD_BANK_LUT_ERR_INVALID_ADDRESS_LOOKUP_TABLE_INDEX (-5)
-
-#define FD_BANK_LUT_ERR_LAST                               (-5)
-
-static inline int
-fd_bank_lut_err_from_runtime_err( int err ) {
-   switch( err ) {
-      case FD_RUNTIME_EXECUTE_SUCCESS:                            return FD_BANK_LUT_ERR_SUCCESS;
-      case FD_RUNTIME_TXN_ERR_ADDRESS_LOOKUP_TABLE_NOT_FOUND:     return FD_BANK_LUT_ERR_ADDRESS_LOOKUP_TABLE_NOT_FOUND;
-      case FD_RUNTIME_TXN_ERR_INVALID_ADDRESS_LOOKUP_TABLE_OWNER: return FD_BANK_LUT_ERR_INVALID_ADDRESS_LOOKUP_TABLE_OWNER;
-      case FD_RUNTIME_TXN_ERR_INVALID_ADDRESS_LOOKUP_TABLE_DATA:  return FD_BANK_LUT_ERR_INVALID_ADDRESS_LOOKUP_TABLE_DATA;
-      case FD_RUNTIME_TXN_ERR_INVALID_ADDRESS_LOOKUP_TABLE_INDEX: return FD_BANK_LUT_ERR_INVALID_ADDRESS_LOOKUP_TABLE_INDEX;
-      default: FD_LOG_ERR(( "Unknown runtime LUT error %d", err ));
-   }
-}
-
 #endif /* HEADER_fd_src_discof_bank_fd_bank_err_h */

src/discof/resolv/fd_resolv_tile.c

@@ -1,12 +1,12 @@
 #include "fd_resolv_tile.h"
 #include "../../disco/fd_txn_m.h"
 #include "../../disco/topo/fd_topo.h"
-#include "../bank/fd_bank_err.h"
 #include "../replay/fd_replay_tile.h"
 #include "generated/fd_resolv_tile_seccomp.h"
 #include "../../disco/metrics/fd_metrics.h"
+#include "../../flamenco/accdb/fd_accdb_sync.h"
+#include "../../flamenco/runtime/fd_alut_interp.h"
 #include "../../flamenco/runtime/fd_system_ids_pp.h"
-#include "../../flamenco/runtime/fd_runtime.h"
 #include "../../flamenco/runtime/fd_bank.h"
 #include "../../util/pod/fd_pod_format.h"
 
@@ -147,7 +147,8 @@ typedef struct {
      rooted bank (after exchanging it for a new rooted bank). */
   fd_banks_t * banks;
   fd_bank_t *  bank;
-  fd_funk_t    funk[1];
+
+  fd_accdb_user_t accdb[1];
 
   fd_stashed_txn_m_t * pool;
   map_chain_t *        map_chain;
@@ -165,6 +166,7 @@ typedef struct {
     ulong blockhash_expired;
     ulong bundle_peer_failure;
     ulong stash[ FD_METRICS_COUNTER_RESOLV_STASH_OPERATION_CNT ];
+    ulong db_race;
   } metrics;
 
   fd_resolv_in_ctx_t in[ 64UL ];
@@ -195,6 +197,7 @@ metrics_write( fd_resolv_ctx_t * ctx ) {
   FD_MCNT_ENUM_COPY( RESOLF, LUT_RESOLVED,                    ctx->metrics.lut );
   FD_MCNT_ENUM_COPY( RESOLF, STASH_OPERATION,                 ctx->metrics.stash );
   FD_MCNT_SET(       RESOLF, TRANSACTION_BUNDLE_PEER_FAILURE, ctx->metrics.bundle_peer_failure );
+  FD_MCNT_SET(       RESOLF, DB_RACES,                        ctx->metrics.db_race );
 }
 
 static int
@@ -241,7 +244,99 @@ during_frag( fd_resolv_ctx_t * ctx,
   }
 }
 
-static inline int
+/* peek_alut reads a single address lookup table from database cache. */
+
+static int
+peek_alut( fd_resolv_ctx_t *  ctx,
+           fd_txn_m_t *       txnm,
+           fd_alut_interp_t * interp,
+           ulong              alut_idx ) {
+  fd_funk_txn_xid_t const xid = { .ul = { fd_bank_slot_get( ctx->bank ), fd_bank_slot_get( ctx->bank ) } };
+
+  ulong ro_indir_cnt_old = interp->ro_indir_cnt;
+  ulong rw_indir_cnt_old = interp->rw_indir_cnt;
+  ulong alut_idx_old     = interp->alut_idx;
+
+  fd_txn_t const * txn         = fd_txn_m_txn_t_const  ( txnm );
+  uchar const *    txn_payload = fd_txn_m_payload_const( txnm );
+  fd_txn_acct_addr_lut_t const * addr_lut =
+      &fd_txn_get_address_tables_const( txn )[ alut_idx ];
+  fd_pubkey_t addr_lut_acc = FD_LOAD( fd_pubkey_t, txn_payload+addr_lut->addr_off );
+
+  int err = 0;
+  for(;;) {
+    fd_accdb_peek_t _peek[1];
+    fd_accdb_peek_t * peek = fd_accdb_peek(
+        ctx->accdb, _peek, &xid, &addr_lut_acc );
+    if( FD_UNLIKELY( !peek ) ) {
+      err = FD_RUNTIME_TXN_ERR_ADDRESS_LOOKUP_TABLE_NOT_FOUND;
+      break;
+    }
+
+    err = fd_alut_interp_next(
+        interp,
+        &addr_lut_acc,
+        fd_accdb_ref_owner     ( peek->acc ),
+        fd_accdb_ref_data_const( peek->acc ),
+        fd_accdb_ref_data_sz   ( peek->acc ) );
+
+    int peek_ok = fd_accdb_peek_test( peek );
+    fd_accdb_peek_drop( peek );
+    if( FD_LIKELY( peek_ok ) ) break;
+
+    /* Restore old interp state and retry */
+    FD_SPIN_PAUSE();
+    ctx->metrics.db_race++;
+    interp->ro_indir_cnt = ro_indir_cnt_old;
+    interp->rw_indir_cnt = rw_indir_cnt_old;
+    interp->alut_idx     = alut_idx_old;
+  }
+
+  return err;
+}
+
+/* peek_aluts reads address lookup tables from database cache.
+   Gracefully recovers from data races and missing accounts. */
+
+static int
+peek_aluts( fd_resolv_ctx_t * ctx,
+            fd_txn_m_t *      txnm ) {
+
+  /* Unpack context */
+  fd_txn_t const *          txn          = fd_txn_m_txn_t_const  ( txnm );
+  uchar const *             txn_payload  = fd_txn_m_payload_const( txnm );
+  ulong const               alut_cnt     = txn->addr_table_lookup_cnt;
+  ulong const               slot         = fd_bank_slot_get( ctx->bank );
+  fd_sysvar_cache_t const * sysvar_cache = fd_bank_sysvar_cache_query( ctx->bank ); FD_TEST( sysvar_cache );
+  fd_slot_hash_t const *    slot_hashes  = fd_sysvar_cache_slot_hashes_join_const( sysvar_cache );
+
+  /* Write indirect addrs into here */
+  fd_acct_addr_t * indir_addrs = fd_txn_m_alut( txnm );
+
+  int err = FD_RUNTIME_EXECUTE_SUCCESS;
+  fd_alut_interp_t interp[1];
+  fd_alut_interp_new( interp, indir_addrs, txn, txn_payload, slot_hashes, slot );
+  for( ulong i=0UL; i<alut_cnt; i++ ) {
+    err = peek_alut( ctx, txnm, interp, i );
+    if( FD_UNLIKELY( err ) ) break;
+  }
+  fd_alut_interp_delete( interp );
+  fd_sysvar_cache_slot_hashes_leave_const( sysvar_cache, slot_hashes );
+
+  ulong ctr_idx;
+  switch( err ) {
+  case FD_RUNTIME_EXECUTE_SUCCESS:                            ctr_idx = FD_METRICS_ENUM_LUT_RESOLVE_RESULT_V_SUCCESS_IDX;               break;
+  case FD_RUNTIME_TXN_ERR_ADDRESS_LOOKUP_TABLE_NOT_FOUND:     ctr_idx = FD_METRICS_ENUM_LUT_RESOLVE_RESULT_V_ACCOUNT_NOT_FOUND_IDX;     break;
+  case FD_RUNTIME_TXN_ERR_INVALID_ADDRESS_LOOKUP_TABLE_OWNER: ctr_idx = FD_METRICS_ENUM_LUT_RESOLVE_RESULT_V_INVALID_ACCOUNT_OWNER_IDX; break;
+  case FD_RUNTIME_TXN_ERR_INVALID_ADDRESS_LOOKUP_TABLE_DATA:  ctr_idx = FD_METRICS_ENUM_LUT_RESOLVE_RESULT_V_INVALID_ACCOUNT_DATA_IDX;  break;
+  case FD_RUNTIME_TXN_ERR_INVALID_ADDRESS_LOOKUP_TABLE_INDEX: ctr_idx = FD_METRICS_ENUM_LUT_RESOLVE_RESULT_V_INVALID_LOOKUP_INDEX_IDX;  break;
+  default:                                                    ctr_idx = FD_METRICS_ENUM_LUT_RESOLVE_RESULT_V_ACCOUNT_UNINITIALIZED_IDX; break;
+  }
+  ctx->metrics.lut[ ctr_idx ]++;
+  return err;
+}
+
+static int
 publish_txn( fd_resolv_ctx_t *          ctx,
              fd_stem_context_t *        stem,
              fd_stashed_txn_m_t const * stashed ) {
@@ -257,24 +352,8 @@ publish_txn( fd_resolv_ctx_t *          ctx,
       FD_MCNT_INC( RESOLF, NO_BANK_DROP, 1 );
       return 0;
     }
-
-    fd_sysvar_cache_t const * sysvar_cache = fd_bank_sysvar_cache_query( ctx->bank );
-    FD_TEST( sysvar_cache );
-
-    fd_funk_txn_xid_t xid = { .ul = { fd_bank_slot_get( ctx->bank ), fd_bank_slot_get( ctx->bank ) } };
-
-    fd_slot_hash_t const * slot_hashes = fd_sysvar_cache_slot_hashes_join_const( sysvar_cache );
-
-    int result = fd_runtime_load_txn_address_lookup_tables( txnt,
-                                                            fd_txn_m_payload( txnm ),
-                                                            ctx->funk,
-                                                            &xid,
-                                                            fd_bank_slot_get( ctx->bank ),
-                                                            slot_hashes,
-                                                            fd_txn_m_alut( txnm ) );
-    fd_sysvar_cache_slot_hashes_leave_const( sysvar_cache, slot_hashes );
-    ctx->metrics.lut[ result ]++;
-    if( FD_UNLIKELY( result ) ) return 0;
+    int err = peek_aluts( ctx, txnm );
+    if( FD_UNLIKELY( err ) ) return 0;
   }
 
   ulong realized_sz = fd_txn_m_realized_footprint( txnm, 1, 1 );
@@ -481,22 +560,7 @@ after_frag( fd_resolv_ctx_t *   ctx,
       return;
     }
 
-    fd_sysvar_cache_t const * sysvar_cache = fd_bank_sysvar_cache_query( ctx->bank );
-    FD_TEST( sysvar_cache );
-    fd_slot_hash_t const * slot_hashes = fd_sysvar_cache_slot_hashes_join_const( sysvar_cache );
-    FD_TEST( slot_hashes );
-
-    fd_funk_txn_xid_t xid = { .ul = { fd_bank_slot_get( ctx->bank ), fd_bank_slot_get( ctx->bank ) } };
-
-    int result = fd_runtime_load_txn_address_lookup_tables( txnt,
-                                                            fd_txn_m_payload( txnm ),
-                                                            ctx->funk,
-                                                            &xid,
-                                                            fd_bank_slot_get( ctx->bank ),
-                                                            slot_hashes,
-                                                            fd_txn_m_alut( txnm ) );
-    fd_sysvar_cache_slot_hashes_leave_const( sysvar_cache, slot_hashes );
-    ctx->metrics.lut[ -fd_bank_lut_err_from_runtime_err( result ) ]++;
+    int result = peek_aluts( ctx, txnm );
     if( FD_UNLIKELY( result ) ) {
       if( FD_UNLIKELY( txnm->block_engine.bundle_id ) ) ctx->bundle_failed = 1;
       return;
@@ -567,7 +631,7 @@ unprivileged_init( fd_topo_t *      topo,
   ctx->out_replay->wmark  = fd_dcache_compact_wmark ( ctx->out_replay->mem, topo->links[ tile->out_link_id[ 1 ] ].dcache, topo->links[ tile->out_link_id[ 1 ] ].mtu );
   ctx->out_replay->chunk  = ctx->out_replay->chunk0;
 
-  FD_TEST( fd_funk_join( ctx->funk, fd_topo_obj_laddr( topo, tile->resolv.funk_obj_id ) ) );
+  FD_TEST( fd_accdb_user_join( ctx->accdb, fd_topo_obj_laddr( topo, tile->resolv.funk_obj_id ) ) );
 
   ulong banks_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "banks" );
   FD_TEST( banks_obj_id!=ULONG_MAX );

src/discof/restore/fd_snapin_tile.c

@@ -620,7 +620,7 @@ handle_control_frag( fd_snapin_tile_t *  ctx,
       ctx->state = FD_SNAPSHOT_STATE_IDLE;
 
       if( ctx->full ) {
-        fd_funk_txn_remove_published( funk );
+        fd_accdb_clear( ctx->accdb_admin );
       } else {
         fd_accdb_cancel( ctx->accdb_admin, ctx->xid );
         fd_funk_txn_xid_copy( ctx->xid, fd_funk_last_publish( funk ) );
@@ -637,7 +637,7 @@ handle_control_frag( fd_snapin_tile_t *  ctx,
       }
       ctx->state = FD_SNAPSHOT_STATE_IDLE;
 
-      fd_funk_txn_xid_t incremental_xid = fd_funk_generate_xid();
+      fd_funk_txn_xid_t incremental_xid = { .ul={ LONG_MAX, LONG_MAX } };
       fd_accdb_attach_child( ctx->accdb_admin, ctx->xid, &incremental_xid );
       fd_funk_txn_xid_copy( ctx->xid, &incremental_xid );
       break;

src/flamenco/accdb/Local.mk

@@ -5,3 +5,6 @@ $(call add-objs,fd_accdb_admin,fd_flamenco)
 # User API
 $(call add-hdrs,fd_accdb_user.h fd_accdb_sync.h)
 $(call add-objs,fd_accdb_user,fd_flamenco)
+
+$(call make-unit-test,test_accdb,test_accdb,fd_flamenco fd_funk fd_util)
+$(call run-unit-test,test_accdb)

src/flamenco/accdb/fd_accdb_admin.c

@@ -443,5 +443,5 @@ fd_accdb_clear( fd_accdb_admin_t * cache ) {
 
 void
 fd_accdb_verify( fd_accdb_admin_t * admin ) {
-  fd_funk_verify( admin->funk );
+  FD_TEST( fd_funk_verify( admin->funk )==FD_FUNK_SUCCESS );
 }

src/flamenco/accdb/fd_accdb_user.c

@@ -218,6 +218,7 @@ fd_accdb_peek( fd_accdb_user_t *         accdb,
     FD_SPIN_PAUSE();
     /* FIXME backoff */
   }
+  if( !rec ) return NULL;
 
   *peek = (fd_accdb_peek_t) {
     .acc = {{