Skip to content

pack/resolv: fix transaction expiration logic #7128

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

pack/resolv: fix transaction expiration logic #7128

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8a42930
fix_expiration_logic
HGuillemet Nov 6, 2025
5d9e3c0
Wrap comment
HGuillemet Nov 7, 2025
9d5e913
Fix blockhash when extra storage is on. Use `fd_txn_p.reference_block…
HGuillemet Nov 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/ballet/txn/fd_txn.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,10 @@
transaction, using fd_txn_parse() verification rules. */
#define FD_TXN_MIN_SERIALIZED_SZ (134UL)

/* FD_TXN_MAX_BLOCK_HEIGHT: Max number of accepted blockhashes before a
transaction expires. */
#define FD_TXN_MAX_BLOCK_HEIGHT 151UL

/* BEGIN Agave limits */

/* "Maximum number of accounts that a transaction may lock.
Expand Down
24 changes: 12 additions & 12 deletions src/disco/bundle/fd_bundle_client.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -462,12 +462,12 @@ fd_bundle_tile_publish_bundle_txn(

fd_txn_m_t * txnm = fd_chunk_to_laddr( ctx->verify_out.mem, ctx->verify_out.chunk );
*txnm = (fd_txn_m_t) {
.reference_slot = 0UL,
.payload_sz = (ushort)txn_sz,
.txn_t_sz = 0U,
.source_ipv4 = source_ipv4,
.source_tpu = FD_TXN_M_TPU_SOURCE_BUNDLE,
.block_engine = {
.reference_block_height = 0UL,
.payload_sz = (ushort)txn_sz,
.txn_t_sz = 0U,
.source_ipv4 = source_ipv4,
.source_tpu = FD_TXN_M_TPU_SOURCE_BUNDLE,
.block_engine = {
.bundle_id = ctx->bundle_seq,
.bundle_txn_cnt = bundle_txn_cnt,
.commission = (uchar)ctx->builder_commission
Expand Down Expand Up @@ -500,12 +500,12 @@ fd_bundle_tile_publish_txn(
) {
fd_txn_m_t * txnm = fd_chunk_to_laddr( ctx->verify_out.mem, ctx->verify_out.chunk );
*txnm = (fd_txn_m_t) {
.reference_slot = 0UL,
.payload_sz = (ushort)txn_sz,
.txn_t_sz = 0U,
.source_ipv4 = source_ipv4,
.source_tpu = FD_TXN_M_TPU_SOURCE_BUNDLE,
.block_engine = {
.reference_block_height = 0UL,
.payload_sz = (ushort)txn_sz,
.txn_t_sz = 0U,
.source_ipv4 = source_ipv4,
.source_tpu = FD_TXN_M_TPU_SOURCE_BUNDLE,
.block_engine = {
.bundle_id = 0UL,
.bundle_txn_cnt = 1UL,
.commission = 0U,
Expand Down
8 changes: 4 additions & 4 deletions src/disco/fd_txn_m.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@
#define FD_TXN_M_TPU_SOURCE_SEND (5UL)

struct fd_txn_m {
/* The computed slot that this transaction is referencing, aka. the
slot number of the reference_blockhash. If it could not be
determined, this will be the current slot. */
ulong reference_slot;
/* The computed block height that this transaction is referencing, aka. the
block height of the slot having the reference_blockhash. If it could not be
determined, this will be block height of the current slot. */
ulong reference_block_height;

ushort payload_sz;

Expand Down
2 changes: 1 addition & 1 deletion src/disco/fd_txn_p.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ struct __attribute__((aligned(64))) fd_txn_p {
uint rebated_cus; /* requested_exec_plus_acct_data_cus-actual used CUs. Pack reads this for CU rebating. */
uint actual_consumed_cus; /* non_execution_cus+real execution CUs+real account data cus. PoH reads this for block CU counting. */
} bank_cu; /* Populated by bank. */
ulong blockhash_slot; /* Slot provided by resolv tile when txn arrives at the pack tile. Used when txn is in extra storage in pack. */
ulong reference_block_height; /* Block height provided by resolv tile when txn arrives at the pack tile. */
};
/* The time that the transaction arrived to the pack tile in ticks. Set by pack and intended to be read from a transaction on a pack->bank link. */
long scheduler_arrival_time_nanos;
Expand Down
9 changes: 4 additions & 5 deletions src/disco/pack/fd_pack.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1284,7 +1284,6 @@ populate_bitsets( fd_pack_t * pack,
int
fd_pack_insert_txn_fini( fd_pack_t * pack,
fd_txn_e_t * txne,
ulong expires_at,
ulong * delete_cnt ) {
*delete_cnt = 0UL;

Expand All @@ -1299,7 +1298,7 @@ fd_pack_insert_txn_fini( fd_pack_t * pack,
accessed with adj_lut[n]. */
fd_acct_addr_t const * alt_adj = ord->txn_e->alt_accts - fd_txn_account_cnt( txn, FD_TXN_ACCT_CAT_IMM );

ord->expires_at = expires_at;
ord->expires_at = txne->txnp->reference_block_height;

int est_result = fd_pack_estimate_rewards_and_compute( txne, ord );
if( FD_UNLIKELY( !est_result ) ) REJECT( ESTIMATION_FAIL );
Expand All @@ -1318,7 +1317,7 @@ fd_pack_insert_txn_fini( fd_pack_t * pack,
}

/* Reject any transactions that have already expired */
if( FD_UNLIKELY( expires_at<pack->expire_before ) ) REJECT( EXPIRED );
if( FD_UNLIKELY( ord->expires_at<pack->expire_before ) ) REJECT( EXPIRED );

int replaces = 0;
/* If it's a durable nonce and we already have one, delete one or the
Expand Down Expand Up @@ -1388,7 +1387,7 @@ fd_pack_insert_txn_fini( fd_pack_t * pack,

if( FD_UNLIKELY( is_durable_nonce ) ) noncemap_ele_insert( pack->noncemap, ord, pack->pool );

fd_pack_expq_t temp[ 1 ] = {{ .expires_at = expires_at, .txn = ord }};
fd_pack_expq_t temp[ 1 ] = {{ .expires_at = ord->expires_at, .txn = ord }};
expq_insert( pack->expiration_q, temp );

if( FD_LIKELY( is_vote ) ) insert_into = pack->pending_votes;
Expand Down Expand Up @@ -1977,7 +1976,7 @@ fd_pack_schedule_impl( fd_pack_t * pack,
/* Copied out to 1280 bytes, which copies some other fields we needed to
copy anyway. */
FD_STATIC_ASSERT( offsetof(fd_txn_p_t, payload_sz )+sizeof(((fd_txn_p_t*)NULL)->payload_sz )<=1280UL, nt_memcpy );
FD_STATIC_ASSERT( offsetof(fd_txn_p_t, blockhash_slot )+sizeof(((fd_txn_p_t*)NULL)->blockhash_slot)<=1280UL, nt_memcpy );
FD_STATIC_ASSERT( offsetof(fd_txn_p_t, reference_block_height )+sizeof(((fd_txn_p_t*)NULL)->reference_block_height )<=1280UL, nt_memcpy );
FD_STATIC_ASSERT( offsetof(fd_txn_p_t, scheduler_arrival_time_nanos )+sizeof(((fd_txn_p_t*)NULL)->scheduler_arrival_time_nanos )<=1280UL, nt_memcpy );
FD_STATIC_ASSERT( offsetof(fd_txn_p_t, source_tpu )+sizeof(((fd_txn_p_t*)NULL)->source_tpu )<=1280UL, nt_memcpy );
FD_STATIC_ASSERT( offsetof(fd_txn_p_t, source_ipv4 )+sizeof(((fd_txn_p_t*)NULL)->source_ipv4 )<=1280UL, nt_memcpy );
Expand Down
6 changes: 3 additions & 3 deletions src/disco/pack/fd_pack.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -333,9 +333,9 @@ FD_STATIC_ASSERT( FD_PACK_INSERT_ACCEPT_NONCE_NONVOTE_REPLACE<FD_PACK_INSERT_RET
returns one of the FD_PACK_INSERT_ACCEPT_* or FD_PACK_INSERT_REJECT_*
codes explained above.
*/
fd_txn_e_t * fd_pack_insert_txn_init ( fd_pack_t * pack );
int fd_pack_insert_txn_fini ( fd_pack_t * pack, fd_txn_e_t * txn, ulong expires_at, ulong * delete_cnt );
void fd_pack_insert_txn_cancel( fd_pack_t * pack, fd_txn_e_t * txn );
fd_txn_e_t * fd_pack_insert_txn_init ( fd_pack_t * pack );
int fd_pack_insert_txn_fini ( fd_pack_t * pack, fd_txn_e_t * txn, ulong * delete_cnt );
void fd_pack_insert_txn_cancel( fd_pack_t * pack, fd_txn_e_t * txn );

/* fd_pack_insert_bundle_{init,fini,cancel} are parallel to the
similarly named fd_pack_insert_txn functions but can be used to
Expand Down
77 changes: 34 additions & 43 deletions src/disco/pack/fd_pack_tile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,6 @@
transactions/microblock, that's 62k txn/sec/bank. */
#define MICROBLOCK_DURATION_NS (0L)

/* There are 151 accepted blockhashes, but those don't include skips.
This check is neither precise nor accurate, but just good enough.
The bank tile does the final check. We give a little margin for a
few percent skip rate. */
#define TRANSACTION_LIFETIME_SLOTS 160UL

/* Time is normally a long, but pack expects a ulong. Add -LONG_MIN to
the time values so that LONG_MIN maps to 0, LONG_MAX maps to
ULONG_MAX, and everything in between maps linearly with a slope of 1.
Expand Down Expand Up @@ -213,10 +207,9 @@ typedef struct {
successful transaction insert. */
long last_successful_insert;

/* highest_observed_slot stores the highest slot number we've seen
from any transaction coming from the resolv tile. When this
increases, we expire old transactions. */
ulong highest_observed_slot;
/* root_block_height stores the block height of the last rooted slot we've seen
coming from the resolv tile. When this increases, we expire old transactions. */
ulong root_block_height;

/* microblock_duration_ns, and wait_duration
respectively scaled to be in ticks instead of nanoseconds */
Expand Down Expand Up @@ -276,7 +269,7 @@ typedef struct {
ulong id;
ulong txn_cnt;
ulong txn_received;
ulong min_blockhash_slot;
ulong min_block_height;
fd_txn_e_t * _txn[ FD_PACK_MAX_TXN_PER_BUNDLE ];
fd_txn_e_t * const * bundle; /* points to _txn when non-NULL */
} current_bundle[1];
Expand Down Expand Up @@ -458,11 +451,9 @@ insert_from_extra( fd_pack_ctx_t * ctx ) {
spot->txnp->scheduler_arrival_time_nanos = insert->txnp->scheduler_arrival_time_nanos;
extra_txn_deq_remove_head( ctx->extra_txn_deq );

ulong blockhash_slot = insert->txnp->blockhash_slot;

ulong deleted;
long insert_duration = -fd_tickcount();
int result = fd_pack_insert_txn_fini( ctx->pack, spot, blockhash_slot, &deleted );
int result = fd_pack_insert_txn_fini( ctx->pack, spot, &deleted );
insert_duration += fd_tickcount();

FD_MCNT_INC( PACK, TRANSACTION_DELETED, deleted );
Expand Down Expand Up @@ -849,29 +840,30 @@ during_frag( fd_pack_ctx_t * ctx,
if( FD_UNLIKELY( chunk<ctx->in[ in_idx ].chunk0 || chunk>ctx->in[ in_idx ].wmark || sz>FD_TPU_RESOLVED_MTU ) )
FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->in[ in_idx ].chunk0, ctx->in[ in_idx ].wmark ));

fd_txn_m_t * txnm = (fd_txn_m_t *)dcache_entry;
ulong payload_sz = txnm->payload_sz;
ulong txn_t_sz = txnm->txn_t_sz;
uint source_ipv4 = txnm->source_ipv4;
uchar source_tpu = txnm->source_tpu;
fd_txn_m_t * txnm = (fd_txn_m_t *)dcache_entry;
ulong payload_sz = txnm->payload_sz;
ulong txn_t_sz = txnm->txn_t_sz;
uint source_ipv4 = txnm->source_ipv4;
uchar source_tpu = txnm->source_tpu;
ulong reference_block_height = txnm->reference_block_height;
FD_TEST( payload_sz<=FD_TPU_MTU );
FD_TEST( txn_t_sz <=FD_TXN_MAX_SZ );
fd_txn_t * txn = fd_txn_m_txn_t( txnm );

ulong addr_table_sz = 32UL*txn->addr_table_adtl_cnt;
FD_TEST( addr_table_sz<=32UL*FD_TXN_ACCT_ADDR_MAX );

if( FD_UNLIKELY( (ctx->leader_slot==ULONG_MAX) & (sig>ctx->highest_observed_slot) ) ) {
/* Using the resolv tile's knowledge of the current slot is a bit
of a hack, since we don't get any info if there are no
transactions and we're not leader. We're actually in exactly
the case where that's okay though. The point of calling
expire_before long before we become leader is so that we don't
drop new but low-fee-paying transactions when pack is clogged
with expired but high-fee-paying transactions. That can only
happen if we are getting transactions. */
ctx->highest_observed_slot = sig;
ulong exp_cnt = fd_pack_expire_before( ctx->pack, fd_ulong_max( ctx->highest_observed_slot, TRANSACTION_LIFETIME_SLOTS )-TRANSACTION_LIFETIME_SLOTS );
if( FD_UNLIKELY( (ctx->leader_slot==ULONG_MAX) & (sig>ctx->root_block_height) ) ) {
/* Using the resolv tile's knowledge of the current last root
block height is a bit of a hack, since we don't get any info if
there are no transactions and we're not leader. We're actually
in exactly the case where that's okay though. The point of
calling expire_before long before we become leader is so that
we don't drop new but low-fee-paying transactions when pack is
clogged with expired but high-fee-paying transactions.
That can only happen if we are getting transactions. */
ctx->root_block_height = sig;
Copy link
Contributor

@mmcgee-jump mmcgee-jump Nov 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This isn't the root block height though? It's the block height of the referenced blockhash, I don't think this logic works

Copy link
Contributor Author

@HGuillemet HGuillemet Nov 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The idea is to pass the root block height as the sig, and use it to discard already buffered txs, and the reference (= blockhash) block height of the sent tx in the reference_block_height of the fd_txn_m.
Why do you think it won't work ?

ulong exp_cnt = fd_pack_expire_before( ctx->pack, fd_ulong_max( ctx->root_block_height, FD_TXN_MAX_BLOCK_HEIGHT )-FD_TXN_MAX_BLOCK_HEIGHT );
FD_MCNT_INC( PACK, TRANSACTION_EXPIRED, exp_cnt );
}

Expand All @@ -886,7 +878,7 @@ during_frag( fd_pack_ctx_t * ctx,
}
ctx->current_bundle->id = bundle_id;
ctx->current_bundle->txn_cnt = txnm->block_engine.bundle_txn_cnt;
ctx->current_bundle->min_blockhash_slot = ULONG_MAX;
ctx->current_bundle->min_block_height = ULONG_MAX;
ctx->current_bundle->txn_received = 0UL;

if( FD_UNLIKELY( ctx->current_bundle->txn_cnt==0UL ) ) {
Expand All @@ -899,8 +891,8 @@ during_frag( fd_pack_ctx_t * ctx,

ctx->current_bundle->bundle = fd_pack_insert_bundle_init( ctx->pack, ctx->current_bundle->_txn, ctx->current_bundle->txn_cnt );
}
ctx->cur_spot = ctx->current_bundle->bundle[ ctx->current_bundle->txn_received ];
ctx->current_bundle->min_blockhash_slot = fd_ulong_min( ctx->current_bundle->min_blockhash_slot, sig );
ctx->cur_spot = ctx->current_bundle->bundle[ ctx->current_bundle->txn_received ];
ctx->current_bundle->min_block_height = fd_ulong_min( ctx->current_bundle->min_block_height, reference_block_height );
} else {
ctx->is_bundle = 0;
#if FD_PACK_USE_EXTRA_STORAGE
Expand All @@ -913,11 +905,7 @@ during_frag( fd_pack_ctx_t * ctx,
FD_MCNT_INC( PACK, TRANSACTION_DROPPED_FROM_EXTRA, 1UL );
}
ctx->cur_spot = extra_txn_deq_peek_tail( extra_txn_deq_insert_tail( ctx->extra_txn_deq ) );
/* We want to store the current time in cur_spot so that we can
track its expiration better. We just stash it in the CU
fields, since those aren't important right now. */
ctx->cur_spot->txnp->blockhash_slot = sig;
ctx->insert_to_extra = 1;
ctx->insert_to_extra = 1;
FD_MCNT_INC( PACK, TRANSACTION_INSERTED_TO_EXTRA, 1UL );
}
#else
Expand All @@ -936,6 +924,10 @@ during_frag( fd_pack_ctx_t * ctx,
ctx->cur_spot->txnp->payload_sz = payload_sz;
ctx->cur_spot->txnp->source_ipv4 = source_ipv4;
ctx->cur_spot->txnp->source_tpu = source_tpu;
/* We just stash the reference block height in the CU
fields, since those aren't important until
fd_pack_insert_txn_fini is called. */
ctx->cur_spot->txnp->reference_block_height = reference_block_height;

break;
}
Expand Down Expand Up @@ -1006,7 +998,7 @@ after_frag( fd_pack_ctx_t * ctx,
}
ctx->leader_slot = leader_slot;

ulong exp_cnt = fd_pack_expire_before( ctx->pack, fd_ulong_max( ctx->leader_slot, TRANSACTION_LIFETIME_SLOTS )-TRANSACTION_LIFETIME_SLOTS );
ulong exp_cnt = fd_pack_expire_before( ctx->pack, fd_ulong_max( ctx->_became_leader->block_height, FD_TXN_MAX_BLOCK_HEIGHT )-FD_TXN_MAX_BLOCK_HEIGHT );
FD_MCNT_INC( PACK, TRANSACTION_EXPIRED, exp_cnt );

ctx->leader_bank = ctx->_became_leader->bank;
Expand Down Expand Up @@ -1076,18 +1068,17 @@ after_frag( fd_pack_ctx_t * ctx,
if( FD_UNLIKELY( ++(ctx->current_bundle->txn_received)==ctx->current_bundle->txn_cnt ) ) {
ulong deleted;
long insert_duration = -fd_tickcount();
int result = fd_pack_insert_bundle_fini( ctx->pack, ctx->current_bundle->bundle, ctx->current_bundle->txn_cnt, ctx->current_bundle->min_blockhash_slot, 0, ctx->blk_engine_cfg, &deleted );
int result = fd_pack_insert_bundle_fini( ctx->pack, ctx->current_bundle->bundle, ctx->current_bundle->txn_cnt, ctx->current_bundle->min_block_height, 0, ctx->blk_engine_cfg, &deleted );
insert_duration += fd_tickcount();
FD_MCNT_INC( PACK, TRANSACTION_DELETED, deleted );
ctx->insert_result[ result + FD_PACK_INSERT_RETVAL_OFF ] += ctx->current_bundle->txn_received;
fd_histf_sample( ctx->insert_duration, (ulong)insert_duration );
ctx->current_bundle->bundle = NULL;
}
} else {
ulong blockhash_slot = sig;
ulong deleted;
long insert_duration = -fd_tickcount();
int result = fd_pack_insert_txn_fini( ctx->pack, ctx->cur_spot, blockhash_slot, &deleted );
int result = fd_pack_insert_txn_fini( ctx->pack, ctx->cur_spot, &deleted );
insert_duration += fd_tickcount();
FD_MCNT_INC( PACK, TRANSACTION_DELETED, deleted );
ctx->insert_result[ result + FD_PACK_INSERT_RETVAL_OFF ]++;
Expand Down Expand Up @@ -1270,7 +1261,7 @@ unprivileged_init( fd_topo_t * topo,
ctx->rng = rng;
ctx->ticks_per_ns = fd_tempo_tick_per_ns( NULL );
ctx->last_successful_insert = 0L;
ctx->highest_observed_slot = 0UL;
ctx->root_block_height = 0UL;
ctx->microblock_duration_ticks = (ulong)(fd_tempo_tick_per_ns( NULL )*(double)MICROBLOCK_DURATION_NS + 0.5);
#if FD_PACK_USE_EXTRA_STORAGE
ctx->insert_to_extra = 0;
Expand Down
6 changes: 4 additions & 2 deletions src/disco/tiles.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ struct __attribute__((aligned(FD_CHUNK_ALIGN))) fd_shred34 {
typedef struct fd_shred34 fd_shred34_t;

struct fd_became_leader {
ulong slot;
ulong slot;
ulong block_height;

/* Start and end time of the slot in nanoseconds (from
fd_log_wallclock()). */
Expand Down Expand Up @@ -105,12 +106,13 @@ typedef struct fd_became_leader fd_became_leader_t;
struct fd_rooted_bank {
void * bank;
ulong slot;
ulong block_height;
};

typedef struct fd_rooted_bank fd_rooted_bank_t;

struct fd_completed_bank {
ulong slot;
ulong block_height;
uchar hash[32];
};

Expand Down
1 change: 1 addition & 0 deletions src/discof/replay/fd_replay_tile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1065,6 +1065,7 @@ publish_root_advanced( fd_replay_tile_t * ctx,

fd_replay_root_advanced_t * msg = fd_chunk_to_laddr( ctx->replay_out->mem, ctx->replay_out->chunk );
msg->bank_idx = bank->idx;
msg->block_height = fd_bank_block_height_get( bank );

fd_stem_publish( stem, ctx->replay_out->idx, REPLAY_SIG_ROOT_ADVANCED, ctx->replay_out->chunk, sizeof(fd_replay_root_advanced_t), 0UL, 0UL, fd_frag_meta_ts_comp( fd_tickcount() ) );
ctx->replay_out->chunk = fd_dcache_compact_next( ctx->replay_out->chunk, sizeof(fd_replay_root_advanced_t), ctx->replay_out->chunk0, ctx->replay_out->wmark );
Expand Down
1 change: 1 addition & 0 deletions src/discof/replay/fd_replay_tile.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ typedef struct fd_replay_slot_completed fd_replay_slot_completed_t;

struct fd_replay_root_advanced {
ulong bank_idx;
ulong block_height;
};

typedef struct fd_replay_root_advanced fd_replay_root_advanced_t;
Expand Down
Loading