Skip to content

Commit 40a602b

Browse files
cali-jumptradingcali-jumptrading
committed
snapshots: lthash
1 parent 3037f95 commit 40a602b

File tree

2 files changed

+81
-2
lines changed

2 files changed

+81
-2
lines changed

‎src/discof/restore/fd_snapin_tile.c‎

Lines changed: 78 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
#include "../../flamenco/runtime/fd_acc_mgr.h"
88
#include "../../flamenco/types/fd_types.h"
99
#include "../../funk/fd_funk.h"
10+
#include "../../ballet/lthash/fd_lthash.h"
11+
#include "../../flamenco/runtime/fd_hashes.h"
1012

1113
#define NAME "snapin"
1214

@@ -31,9 +33,23 @@ struct fd_snapin_tile {
3133
fd_funk_txn_t * funk_txn;
3234
uchar * acc_data;
3335

34-
fd_stem_context_t * stem;
36+
fd_stem_context_t * stem;
3537
fd_snapshot_parser_t * ssparse;
3638

39+
struct {
40+
41+
struct {
42+
fd_lthash_value_t lthash;
43+
fd_lthash_value_t manifest_lthash;
44+
} full;
45+
46+
struct {
47+
fd_lthash_value_t lthash;
48+
fd_lthash_value_t manifest_lthash;
49+
} incremental;
50+
51+
} lthash_info;
52+
3753
struct {
3854
ulong full_bytes_read;
3955
ulong incremental_bytes_read;
@@ -95,6 +111,11 @@ manifest_cb( void * _ctx,
95111
fd_snapin_tile_t * ctx = (fd_snapin_tile_t*)_ctx;
96112

97113
ulong sz = sizeof(fd_snapshot_manifest_t)+manifest_sz;
114+
115+
fd_snapshot_manifest_t * manifest = (fd_snapshot_manifest_t * )fd_chunk_to_laddr_const( ctx->manifest_out.wksp, ctx->manifest_out.chunk );
116+
uchar * manifest_lthash = ctx->full ? ctx->lthash_info.full.manifest_lthash.bytes : ctx->lthash_info.incremental.manifest_lthash.bytes;
117+
fd_memcpy( manifest_lthash, manifest->accounts_lthash, sizeof(fd_lthash_value_t) );
118+
98119
FD_TEST( sz<=ctx->manifest_out.mtu );
99120
ulong sig = ctx->full ? fd_ssmsg_sig( FD_SSMSG_MANIFEST_FULL, manifest_sz ) :
100121
fd_ssmsg_sig( FD_SSMSG_MANIFEST_INCREMENTAL, manifest_sz );
@@ -117,6 +138,19 @@ is_duplicate_account( fd_snapin_tile_t * ctx,
117138
/* TODO: Reaching here means the existing value is a duplicate
118139
account. We need to hash the existing account and subtract that
119140
hash from the running lthash. */
141+
fd_lthash_value_t old_account_lthash[1];
142+
fd_lthash_value_t * lthash = ctx->full ? &ctx->lthash_info.full.lthash : &ctx->lthash_info.incremental.lthash;
143+
fd_hashes_account_lthash( (fd_pubkey_t*)account_pubkey,
144+
rec_meta,
145+
fd_account_meta_get_data_const( rec_meta ),
146+
old_account_lthash );
147+
FD_LOG_WARNING(("subtracting old account hash %s for pubkey %s from lthash %s",
148+
FD_LTHASH_ENC_32_ALLOCA( old_account_lthash ),
149+
FD_BASE58_ENC_32_ALLOCA( (fd_pubkey_t*)account_pubkey ),
150+
FD_LTHASH_ENC_32_ALLOCA( lthash )));
151+
fd_lthash_sub( lthash, old_account_lthash );
152+
FD_LOG_WARNING(("resulting lthash %s",
153+
FD_LTHASH_ENC_32_ALLOCA( lthash )));
120154
}
121155

122156
return 0;
@@ -151,6 +185,21 @@ account_cb( void * _ctx,
151185
ctx->acc_data = fd_txn_account_get_data_mut( rec );
152186
ctx->metrics.accounts_inserted++;
153187
fd_txn_account_mutable_fini( rec, ctx->funk, ctx->funk_txn, &prepare );
188+
189+
fd_lthash_value_t new_account_lthash[1];
190+
fd_lthash_value_t * lthash = ctx->full ? &ctx->lthash_info.full.lthash : &ctx->lthash_info.incremental.lthash;
191+
fd_hashes_account_lthash( (fd_pubkey_t*)hdr->meta.pubkey,
192+
fd_txn_account_get_meta( rec ),
193+
fd_txn_account_get_data( rec ),
194+
new_account_lthash );
195+
FD_LOG_WARNING(("adding new account hash %s for pubkey %s from lthash %s",
196+
FD_LTHASH_ENC_32_ALLOCA( new_account_lthash ),
197+
FD_BASE58_ENC_32_ALLOCA( (fd_pubkey_t*)hdr->meta.pubkey ),
198+
FD_LTHASH_ENC_32_ALLOCA( lthash )));
199+
fd_lthash_add( lthash, new_account_lthash );
200+
FD_LOG_WARNING(("resulting lthash %s",
201+
FD_LTHASH_ENC_32_ALLOCA( lthash )));
202+
154203
}
155204

156205
static void
@@ -220,13 +269,19 @@ handle_control_frag( fd_snapin_tile_t * ctx,
220269
ctx->full = 1;
221270
fd_snapshot_parser_reset( ctx->ssparse, fd_chunk_to_laddr( ctx->manifest_out.wksp, ctx->manifest_out.chunk ), ctx->manifest_out.mtu );
222271
fd_funk_txn_cancel_root( ctx->funk );
272+
fd_lthash_zero( &ctx->lthash_info.full.lthash );
273+
fd_lthash_zero( &ctx->lthash_info.incremental.lthash );
274+
fd_lthash_zero( &ctx->lthash_info.full.manifest_lthash );
275+
fd_lthash_zero( &ctx->lthash_info.incremental.manifest_lthash );
223276
ctx->state = FD_SNAPIN_STATE_LOADING;
224277
break;
225278
case FD_SNAPSHOT_MSG_CTRL_RESET_INCREMENTAL:
226279
ctx->full = 0;
227280
fd_snapshot_parser_reset( ctx->ssparse, fd_chunk_to_laddr( ctx->manifest_out.wksp, ctx->manifest_out.chunk ), ctx->manifest_out.mtu );
228281
if( FD_UNLIKELY( !ctx->funk_txn ) ) fd_funk_txn_cancel_root( ctx->funk );
229282
else fd_funk_txn_cancel( ctx->funk, ctx->funk_txn, 0 );
283+
fd_lthash_zero( &ctx->lthash_info.incremental.lthash );
284+
fd_lthash_zero( &ctx->lthash_info.incremental.manifest_lthash );
230285
ctx->state = FD_SNAPIN_STATE_LOADING;
231286
break;
232287
case FD_SNAPSHOT_MSG_CTRL_EOF_FULL:
@@ -237,6 +292,14 @@ handle_control_frag( fd_snapin_tile_t * ctx,
237292
break;
238293
}
239294

295+
if( FD_UNLIKELY( memcmp( ctx->lthash_info.full.lthash.bytes, ctx->lthash_info.full.manifest_lthash.bytes, sizeof(fd_lthash_value_t) ) ) ) {
296+
FD_LOG_WARNING(( "calculated accounts lthash %s does not match accounts lthash %s in snapshot manifest",
297+
FD_LTHASH_ENC_32_ALLOCA( &ctx->lthash_info.full.lthash ),
298+
FD_LTHASH_ENC_32_ALLOCA( &ctx->lthash_info.full.manifest_lthash ) ));
299+
transition_malformed( ctx, stem );
300+
break;
301+
}
302+
240303
fd_snapshot_parser_reset( ctx->ssparse, fd_chunk_to_laddr( ctx->manifest_out.wksp, ctx->manifest_out.chunk ), ctx->manifest_out.mtu );
241304

242305
fd_funk_txn_xid_t incremental_xid = fd_funk_generate_xid();
@@ -251,6 +314,15 @@ handle_control_frag( fd_snapin_tile_t * ctx,
251314
break;
252315
}
253316

317+
if( FD_UNLIKELY( memcmp( ctx->lthash_info.full.lthash.bytes, ctx->lthash_info.full.manifest_lthash.bytes, sizeof(fd_lthash_value_t) ) ||
318+
memcmp( ctx->lthash_info.incremental.lthash.bytes, ctx->lthash_info.incremental.manifest_lthash.bytes, sizeof(fd_lthash_value_t) ) ) ) {
319+
FD_LOG_WARNING(( "calculated accounts lthash %s does not match accounts lthash %s in snapshot manifest",
320+
FD_LTHASH_ENC_32_ALLOCA( &ctx->lthash_info.full.lthash ),
321+
FD_LTHASH_ENC_32_ALLOCA( &ctx->lthash_info.full.manifest_lthash ) ));
322+
transition_malformed( ctx, stem );
323+
break;
324+
}
325+
254326
if( FD_LIKELY( ctx->funk_txn ) ) fd_funk_txn_publish_into_parent( ctx->funk, ctx->funk_txn, 0 );
255327
fd_stem_publish( stem, 0UL, fd_ssmsg_sig( FD_SSMSG_DONE, 0UL ), 0UL, 0UL, 0UL, 0UL, 0UL );
256328
break;
@@ -346,6 +418,11 @@ unprivileged_init( fd_topo_t * topo,
346418
ctx->in.chunk0 = fd_dcache_compact_chunk0( ctx->in.wksp, in_link->dcache );
347419
ctx->in.wmark = fd_dcache_compact_wmark( ctx->in.wksp, in_link->dcache, in_link->mtu );
348420
ctx->in.mtu = in_link->mtu;
421+
422+
fd_lthash_zero( &ctx->lthash_info.full.lthash );
423+
fd_lthash_zero( &ctx->lthash_info.incremental.lthash );
424+
fd_lthash_zero( &ctx->lthash_info.full.manifest_lthash );
425+
fd_lthash_zero( &ctx->lthash_info.incremental.manifest_lthash );
349426
}
350427

351428
#define STEM_BURST 2UL /* For control fragments, one acknowledgement, and one malformed message */

‎src/discof/restore/utils/fd_ssping.c‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,9 @@ fd_ssping_add( fd_ssping_t * ssping,
231231
static inline void
232232
remove_ping_fd( fd_ssping_t * ssping,
233233
ulong idx ) {
234-
FD_TEST( idx<ssping->fds_len );
234+
if( idx>=ssping->fds_len ) {
235+
FD_LOG_ERR(("idx %lu out of bounds for fds_len %lu", idx, ssping->fds_len));
236+
}
235237

236238
if( FD_UNLIKELY( ssping->fds_len==1UL ) ) {
237239
ssping->fds_len = 0UL;

0 commit comments

Comments
 (0)