restore: initial vinyl integration

Author: riptl

src/app/firedancer-dev/Local.mk

@@ -19,7 +19,7 @@ $(call add-objs,commands/ipecho_server,fd_firedancer_dev)
 $(call add-objs,commands/gossip_dump,fd_firedancer_dev)
 
 ifdef FD_HAS_SSE
-$(call make-bin,firedancer-dev,main,fd_firedancer_dev fd_firedancer fddev_shared fdctl_shared fdctl_platform fd_discof fd_disco fd_choreo fd_flamenco fd_funk fd_quic fd_tls fd_reedsol fd_waltz fd_tango fd_ballet fd_util firedancer_version,$(SECP256K1_LIBS) $(ROCKSDB_LIBS) $(OPENSSL_LIBS))
+$(call make-bin,firedancer-dev,main,fd_firedancer_dev fd_firedancer fddev_shared fdctl_shared fdctl_platform fd_discof fd_disco fd_choreo fd_flamenco fd_vinyl fd_funk fd_quic fd_tls fd_reedsol fd_waltz fd_tango fd_ballet fd_util firedancer_version,$(SECP256K1_LIBS) $(ROCKSDB_LIBS) $(OPENSSL_LIBS))
 endif
 
 # $(call make-integration-test,test_fddev,tests/test_fddev,fd_fddev fd_fdctl fddev_shared fdctl_shared fdctl_platform fd_discof fd_disco fd_choreo fd_flamenco fd_funk fd_quic fd_tls fd_reedsol fd_waltz fd_tango fd_ballet fd_util, $(SECP256K1_LIBS))

src/app/firedancer-dev/commands/backtest.c

@@ -49,6 +49,7 @@ backtest_topo( config_t * config ) {
   ulong exec_tile_cnt   = config->firedancer.layout.exec_tile_count;
 
   int disable_snap_loader = !config->gossip.entrypoints_cnt;
+  int snap_vinyl          = !!config->firedancer.vinyl.enabled;
   int solcap_enabled      = strlen( config->capture.solcap_capture )>0;
 
   fd_topo_t * topo = { fd_topob_new( &config->topo, config->name ) };
@@ -88,6 +89,10 @@ backtest_topo( config_t * config ) {
       config->firedancer.runtime.program_cache.heap_size_mib<<20 );
   fd_topob_tile_uses( topo, replay_tile, progcache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
 
+  if( snap_vinyl ) {
+    setup_topo_vinyl( topo, &config->firedancer );
+  }
+
   /**********************************************************************/
   /* Add the executor tiles to topo                                     */
   /**********************************************************************/
@@ -276,7 +281,15 @@ backtest_topo( config_t * config ) {
       fd_ulong_pow2_up( FD_PACK_MAX_TXNCACHE_TXN_PER_SLOT ) );
   fd_topob_tile_uses( topo, replay_tile, txncache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
   if( FD_LIKELY( !disable_snap_loader ) ) {
-    fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "snapin", 0UL ) ], txncache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
+    fd_topob_tile_uses( topo, snapin_tile, txncache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
+    if( snap_vinyl ) {
+      ulong vinyl_map_obj_id  = fd_pod_query_ulong( topo->props, "vinyl.meta_map",  ULONG_MAX ); FD_TEST( vinyl_map_obj_id !=ULONG_MAX );
+      ulong vinyl_pool_obj_id = fd_pod_query_ulong( topo->props, "vinyl.meta_pool", ULONG_MAX ); FD_TEST( vinyl_pool_obj_id!=ULONG_MAX );
+      fd_topo_obj_t * vinyl_map_obj  = &topo->objs[ vinyl_map_obj_id ];
+      fd_topo_obj_t * vinyl_pool_obj = &topo->objs[ vinyl_pool_obj_id ];
+      fd_topob_tile_uses( topo, snapin_tile, vinyl_map_obj,  FD_SHMEM_JOIN_MODE_READ_WRITE );
+      fd_topob_tile_uses( topo, snapin_tile, vinyl_pool_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
+    }
   }
   for( ulong i=0UL; i<exec_tile_cnt; i++ ) {
     fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "exec", i ) ], txncache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );

src/app/firedancer-dev/commands/snapshot_load.c

@@ -43,6 +43,10 @@ snapshot_load_topo( config_t *     config,
       config->firedancer.funk.max_database_transactions,
       config->firedancer.funk.heap_size_gib );
 
+  if( config->firedancer.vinyl.enabled ) {
+    setup_topo_vinyl( topo, &config->firedancer );
+  }
+
   static ushort tile_to_cpu[ FD_TILE_MAX ] = {0};
   if( args->snapshot_load.tile_cpus[0] ) {
     ulong cpu_cnt = fd_tile_private_cpus_parse( args->snapshot_load.tile_cpus, tile_to_cpu );
@@ -105,6 +109,16 @@ snapshot_load_topo( config_t *     config,
   fd_topob_tile_uses( topo, snapin_tile, txncache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
   snapin_tile->snapin.funk_obj_id     = funk_obj->id;
   snapin_tile->snapin.txncache_obj_id = txncache_obj->id;
+  if( config->firedancer.vinyl.enabled ) {
+    ulong vinyl_map_obj_id  = fd_pod_query_ulong( topo->props, "vinyl.meta_map",  ULONG_MAX ); FD_TEST( vinyl_map_obj_id !=ULONG_MAX );
+    ulong vinyl_pool_obj_id = fd_pod_query_ulong( topo->props, "vinyl.meta_pool", ULONG_MAX ); FD_TEST( vinyl_pool_obj_id!=ULONG_MAX );
+
+    fd_topo_obj_t * vinyl_map_obj  = &topo->objs[ vinyl_map_obj_id ];
+    fd_topo_obj_t * vinyl_pool_obj = &topo->objs[ vinyl_pool_obj_id ];
+
+    fd_topob_tile_uses( topo, snapin_tile, vinyl_map_obj,  FD_SHMEM_JOIN_MODE_READ_WRITE );
+    fd_topob_tile_uses( topo, snapin_tile, vinyl_pool_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
+  }
 
   snapin_tile->snapin.max_live_slots  = config->firedancer.runtime.max_live_slots;
 

src/app/firedancer-dev/main.c

@@ -25,6 +25,9 @@ extern fd_topo_obj_callbacks_t fd_obj_cb_banks;
 extern fd_topo_obj_callbacks_t fd_obj_cb_funk;
 extern fd_topo_obj_callbacks_t fd_obj_cb_bank_hash_cmp;
 
+extern fd_topo_obj_callbacks_t fd_obj_cb_vinyl_meta;
+extern fd_topo_obj_callbacks_t fd_obj_cb_vinyl_meta_ele;
+
 fd_topo_obj_callbacks_t * CALLBACKS[] = {
   &fd_obj_cb_mcache,
   &fd_obj_cb_dcache,
@@ -42,6 +45,8 @@ fd_topo_obj_callbacks_t * CALLBACKS[] = {
   &fd_obj_cb_banks,
   &fd_obj_cb_funk,
   &fd_obj_cb_bank_hash_cmp,
+  &fd_obj_cb_vinyl_meta,
+  &fd_obj_cb_vinyl_meta_ele,
   NULL,
 };
 

src/app/firedancer/Local.mk

@@ -34,17 +34,18 @@ $(OBJDIR)/obj/app/firedancer/version.d: src/app/firedancer/version.h
 # firedancer core
 $(call add-objs,topology,fd_firedancer)
 $(call add-objs,config,fd_firedancer)
-$(call add-objs,callbacks,fd_firedancer)
+$(call add-objs,callbacks callbacks_vinyl,fd_firedancer)
 
 # commands
 $(call add-objs,commands/shred_version,fd_firedancer)
+$(call add-objs,commands/configure/vinyl,fd_firedancer)
 
 # version
 $(call make-lib,firedancer_version)
 $(call add-objs,version,firedancer_version)
 
 ifdef FD_HAS_SSE
-$(call make-bin,firedancer,main,fd_firedancer fdctl_shared fdctl_platform fd_discof fd_disco fd_choreo fd_flamenco fd_funk fd_quic fd_tls fd_reedsol fd_waltz fd_tango fd_ballet fd_util firedancer_version,$(SECP256K1_LIBS) $(OPENSSL_LIBS))
+$(call make-bin,firedancer,main,fd_firedancer fdctl_shared fdctl_platform fd_discof fd_disco fd_choreo fd_flamenco fd_vinyl fd_funk fd_quic fd_tls fd_reedsol fd_waltz fd_tango fd_ballet fd_util firedancer_version,$(SECP256K1_LIBS) $(OPENSSL_LIBS))
 endif
 
 else

src/app/firedancer/callbacks.c

@@ -101,6 +101,34 @@ fd_topo_obj_callbacks_t fd_obj_cb_funk = {
   .new       = funk_new,
 };
 
+/* cnc: a tile admin message queue */
+
+static ulong
+cnc_align( fd_topo_t const *     topo,
+           fd_topo_obj_t const * obj ) {
+  (void)topo; (void)obj;
+  return fd_cnc_align();
+}
+
+static ulong
+cnc_footprint( fd_topo_t const *     topo,
+               fd_topo_obj_t const * obj ) {
+  return fd_cnc_footprint( VAL("app_sz") );
+}
+
+static void
+cnc_new( fd_topo_t const *     topo,
+         fd_topo_obj_t const * obj ) {
+  FD_TEST( fd_cnc_new( fd_topo_obj_laddr( topo, obj->id ), VAL("app_sz"), VAL("type"), fd_log_wallclock() ) );
+}
+
+fd_topo_obj_callbacks_t fd_obj_cb_cnc = {
+  .name      = "cnc",
+  .footprint = cnc_footprint,
+  .align     = cnc_align,
+  .new       = cnc_new,
+};
+
 static ulong
 fec_sets_footprint( fd_topo_t const *     topo,
                     fd_topo_obj_t const * obj ) {

src/app/firedancer/callbacks_vinyl.c

@@ -0,0 +1,84 @@
+#include "../../vinyl/fd_vinyl.h"
+#include "../../disco/topo/fd_topo.h"
+#include "../../util/pod/fd_pod_format.h"
+
+#define VAL(name) (__extension__({                                                             \
+  ulong __x = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.%s", obj->id, name );      \
+  if( FD_UNLIKELY( __x==ULONG_MAX ) ) FD_LOG_ERR(( "obj.%lu.%s was not set", obj->id, name )); \
+  __x; }))
+
+
+/* vinyl_meta: a shared memory separately chained hash map */
+
+static ulong
+vinyl_meta_align( fd_topo_t const *     topo,
+                  fd_topo_obj_t const * obj ) {
+  (void)topo; (void)obj;
+  return fd_vinyl_meta_align();
+}
+
+static ulong
+vinyl_meta_footprint( fd_topo_t const *     topo,
+                      fd_topo_obj_t const * obj ) {
+  return fd_vinyl_meta_footprint( VAL("ele_max"), VAL("lock_cnt"), VAL("probe_max") );
+}
+
+static void
+vinyl_meta_new( fd_topo_t const *     topo,
+                fd_topo_obj_t const * obj ) {
+  FD_TEST( fd_vinyl_meta_new( fd_topo_obj_laddr( topo, obj->id ), VAL("ele_max"), VAL("lock_cnt"), VAL("probe_max"), VAL("seed") ) );
+}
+
+fd_topo_obj_callbacks_t fd_obj_cb_vinyl_meta = {
+  .name      = "vinyl_meta",
+  .footprint = vinyl_meta_footprint,
+  .align     = vinyl_meta_align,
+  .new       = vinyl_meta_new,
+};
+
+/* vinyl_meta_ele: hash map elements of vinyl_meta */
+
+static ulong
+vinyl_meta_ele_align( fd_topo_t const *     topo,
+                      fd_topo_obj_t const * obj ) {
+  (void)topo; (void)obj;
+  return fd_ulong_max( alignof(fd_vinyl_meta_ele_t), 128UL );
+}
+
+static ulong
+vinyl_meta_ele_footprint( fd_topo_t const *     topo,
+                          fd_topo_obj_t const * obj ) {
+  return fd_ulong_align_up( sizeof(fd_vinyl_meta_ele_t) * VAL("cnt"), vinyl_meta_ele_align( topo, obj ) );
+}
+
+static void
+vinyl_meta_ele_new( fd_topo_t const *     topo,
+                    fd_topo_obj_t const * obj ) {
+  /* On Zen 4:
+     - non-temporal wide stores are fastest
+     - scattering 8 byte writes is slower
+     - memset is slowest */
+
+# if FD_HAS_AVX512
+  uchar * m0 = fd_topo_obj_laddr( topo, obj->id );
+  uchar * m1 = m0 + vinyl_meta_ele_footprint( topo, obj );
+  __m512i zero = _mm512_setzero_si512();
+  for( uchar * m=m0; m<m1; m+=64 ) {
+    _mm512_stream_si512( (__m512i *)m, zero );
+  }
+  _mm_sfence();
+# else
+  fd_vinyl_meta_ele_t * ele = fd_topo_obj_laddr( topo, obj->id );
+  ulong cnt = VAL("cnt");
+  for( ulong i=0UL; i<cnt; i++ ) {
+    ele[ i ].phdr.ctl = 0UL;
+  }
+# endif
+}
+
+fd_topo_obj_callbacks_t fd_obj_cb_vinyl_meta_ele = {
+  .name      = "vinyl_meta_e",
+  .footprint = vinyl_meta_ele_footprint,
+  .align     = vinyl_meta_ele_align,
+  .new       = vinyl_meta_ele_new,
+};

src/app/firedancer/commands/configure/vinyl.c

@@ -0,0 +1,75 @@
+#include "../../../shared/commands/configure/configure.h"
+#include "../../../platform/fd_file_util.h"
+
+#include <errno.h>
+#include <fcntl.h>    /* open */
+#include <unistd.h>   /* fchown, close */
+#include <sys/stat.h> /* fchmod */
+
+static int
+enabled( config_t const * config ) {
+  return !!config->firedancer.vinyl.enabled;
+}
+
+static void
+init( config_t const * config ) {
+  if( FD_UNLIKELY( -1==fd_file_util_mkdir_all( config->paths.accounts, config->uid, config->gid, 0 ) ) ) {
+    FD_LOG_ERR(( "fd_file_util_mkdir_all(`%s`) failed (%i-%s)", config->paths.accounts, errno, fd_io_strerror( errno ) ));
+  }
+
+  int vinyl_fd = open( config->paths.accounts, O_RDWR|O_CREAT|O_TRUNC|O_CLOEXEC, S_IRUSR|S_IWUSR );
+  if( FD_UNLIKELY( vinyl_fd<0 ) ) {
+    FD_LOG_ERR(( "open(`%s`,O_RDWR|O_CREAT|O_CLOEXEC,S_IRUSR|S_IWUSR) failed (%i-%s)", config->paths.accounts, errno, fd_io_strerror( errno ) ));
+  }
+
+  if( FD_UNLIKELY( fchown( vinyl_fd, config->uid, config->gid )<0 ) ) {
+    FD_LOG_ERR(( "chown(`%s`,%u:%u) failed (%i-%s)", config->paths.accounts, config->uid, config->gid, errno, fd_io_strerror( errno ) ));
+  }
+
+  if( FD_UNLIKELY( fchmod( vinyl_fd, S_IRUSR|S_IWUSR )<0 ) ) {
+    FD_LOG_ERR(( "chmod(`%s`,S_IRUSR|S_IWUSR) failed (%i-%s)", config->paths.accounts, errno, fd_io_strerror( errno ) ));
+  }
+
+  ulong bstream_sz = config->firedancer.vinyl.file_size_gib<<30;
+  if( FD_UNLIKELY( 0!=ftruncate( vinyl_fd, (long)bstream_sz ) ) ) {
+    FD_LOG_ERR(( "ftruncate(`%s`,%lu bytes) failed (%i-%s)", config->paths.accounts, bstream_sz, errno, fd_io_strerror( errno ) ));
+  }
+
+  if( FD_UNLIKELY( close( vinyl_fd )<0 ) ) {
+    FD_LOG_ERR(( "close(`%s`) failed (%i-%s)", config->paths.accounts, errno, fd_io_strerror( errno ) ));
+  }
+}
+
+static int
+fini( config_t const * config,
+      int              pre_init ) {
+  (void)pre_init;
+  if( FD_UNLIKELY( unlink( config->paths.accounts )<0 ) ) {
+    FD_LOG_ERR(( "unlink(`%s`) failed (%i-%s)", config->paths.accounts, errno, fd_io_strerror( errno ) ));
+  }
+  return 1;
+}
+
+static configure_result_t
+check( config_t const * config ) {
+  struct stat st;
+  if( FD_UNLIKELY( 0!=stat( config->paths.accounts, &st ) ) ) {
+    if( errno==ENOENT ) NOT_CONFIGURED( "`%s` does not exist", config->paths.accounts );
+    else                NOT_CONFIGURED( "stat(`%s`) failed (%i-%s)", config->paths.accounts, errno, fd_io_strerror( errno ) );
+  }
+
+  ulong bstream_sz = config->firedancer.vinyl.file_size_gib<<30;
+  if( FD_UNLIKELY( (ulong)st.st_size!=bstream_sz ) )
+    NOT_CONFIGURED( "`%s` needs to be resized (have %lu bytes, want %lu bytes)", config->paths.accounts, (ulong)st.st_size, bstream_sz );
+
+  CHECK( check_file( config->paths.accounts, config->uid, config->gid, S_IFREG | S_IRUSR | S_IWUSR ) );
+  CONFIGURE_OK();
+}
+
+configure_stage_t fd_cfg_stage_vinyl = {
+  .name    = "vinyl",
+  .enabled = enabled,
+  .init    = init,
+  .fini    = fini,
+  .check   = check,
+};

src/app/firedancer/config/default.toml

@@ -155,6 +155,10 @@ user = ""
     # Firedancer instance.
     genesis = ""
 
+    # Absolute path to the file backing the vinyl account database.
+    # Only used if [vinyl.enabled] is set.
+    accounts = ""
+
 # Firedancer logs to two places by default: stderr and a logfile.
 # stdout is not used for logging, and will only be used to print command
 # output or boot errors.  Messages to "stderr" are abbreviated and not
@@ -466,6 +470,8 @@ user = ""
 # This section configures the "funk" account database.  Currently, funk
 # stores all Solana accounts.  In future versions of Firedancer, most
 # accounts will be offloaded to the "groove" database.
+#
+# FIXME rename 'funk' to human-readable name like 'accounts.live_db'
 [funk]
     # The size of the funk heap in gigabytes.  This value must be large
     # enough to store all Solana accounts uncompressed.
@@ -486,6 +492,12 @@ user = ""
     # setting.
     max_database_transactions = 2048
 
+# FIXME rename 'vinyl' to human-readable name like 'accounts.root_db'
+[vinyl]
+    enabled = false
+    max_account_records = 30_000_000
+    file_size_gib = 16
+
 [runtime]
     max_vote_accounts = 2000000
 

src/app/firedancer/main.c

@@ -43,13 +43,16 @@ fd_topo_obj_callbacks_t * CALLBACKS[] = {
   NULL,
 };
 
+extern configure_stage_t fd_cfg_stage_vinyl;
+
 configure_stage_t * STAGES[] = {
   &fd_cfg_stage_hugetlbfs,
   &fd_cfg_stage_sysctl,
   &fd_cfg_stage_ethtool_channels,
   &fd_cfg_stage_ethtool_offloads,
   &fd_cfg_stage_ethtool_loopback,
   &fd_cfg_stage_snapshots,
+  &fd_cfg_stage_vinyl,
   NULL,
 };